Merge master and resolve conflict

Gemfile.lock

@@ -62,8 +62,10 @@ PATH
       ruby_smb (= 0.0.18)
       rubyntlm
       rubyzip
+      sinatra
       sqlite3
       sshkey
+      thin
       tzinfo
       tzinfo-data
       windows_error
@@ -114,10 +116,12 @@ GEM
     coderay (1.1.2)
     concurrent-ruby (1.0.5)
     crass (1.0.3)
+    daemons (1.2.4)
     diff-lcs (1.3)
     dnsruby (1.60.2)
     docile (1.3.0)
     erubis (2.7.0)
+    eventmachine (1.2.3)
     factory_girl (4.9.0)
       activesupport (>= 3.0.0)
     factory_girl_rails (4.9.0)
@@ -231,6 +235,8 @@ GEM
       method_source (~> 0.9.0)
     public_suffix (3.0.2)
     rack (1.6.9)
+    rack-protection (1.5.3)
+      rack
     rack-test (0.6.3)
       rack (>= 1.0)
     rails-deprecated_sanitizer (1.0.3)
@@ -342,10 +348,19 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    sinatra (1.4.8)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
     sqlite3 (1.3.13)
     sshkey (1.9.0)
+    thin (1.7.1)
+      daemons (~> 1.0, >= 1.0.9)
+      eventmachine (~> 1.0, >= 1.0.4)
+      rack (>= 1, < 3)
     thor (0.20.0)
     thread_safe (0.3.6)
+    tilt (2.0.7)
     timecop (0.9.1)
     ttfunk (1.5.1)
     tzinfo (1.2.5)

lib/metasploit/framework/data_service.rb

@@ -0,0 +1,62 @@
+require 'metasploit/framework/data_service/stubs/host_data_service'
+require 'metasploit/framework/data_service/stubs/vuln_data_service'
+require 'metasploit/framework/data_service/stubs/event_data_service'
+require 'metasploit/framework/data_service/stubs/workspace_data_service'
+require 'metasploit/framework/data_service/stubs/note_data_service'
+require 'metasploit/framework/data_service/stubs/web_data_service'
+require 'metasploit/framework/data_service/stubs/service_data_service'
+require 'metasploit/framework/data_service/stubs/session_data_service'
+require 'metasploit/framework/data_service/stubs/exploit_data_service'
+require 'metasploit/framework/data_service/stubs/loot_data_service'
+
+#
+# All data service implementations should include this module to ensure proper implementation
+#
+module Metasploit
+module Framework
+module DataService
+  include HostDataService
+  include EventDataService
+  include VulnDataService
+  include WorkspaceDataService
+  include WebDataService
+  include NoteDataService
+  include ServiceDataService
+  include SessionDataService
+  include ExploitDataService
+  include LootDataService
+
+  def name
+    raise 'DataLService#name is not implemented';
+  end
+
+  def active
+    raise 'DataLService#active is not implemented';
+  end
+
+  #
+  # Hold metadata about a data service
+  #
+  class Metadata
+    attr_reader :id
+    attr_reader :name
+    attr_reader :active
+
+    def initialize (id, name, active)
+      self.id = id
+      self.name = name
+      self.active = active
+    end
+
+    #######
+    private
+    #######
+
+    attr_writer :id
+    attr_writer :name
+    attr_writer :active
+
+  end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/core.rb

@@ -0,0 +1,163 @@
+require 'open3'
+require 'rex/ui'
+require 'rex/logging'
+require 'metasploit/framework/data_service/remote/http/core'
+require 'metasploit/framework/data_service/proxy/data_proxy_auto_loader'
+
+#
+# Holds references to data services (@see Metasploit::Framework::DataService)
+# and forwards data to the implementation set as current.
+#
+module Metasploit
+module Framework
+module DataService
+class DataProxy
+  include DataProxyAutoLoader
+
+  attr_reader :usable
+
+  def initialize(opts = {})
+    @data_services = {}
+    @data_service_id = 0
+    @usable = false
+    setup(opts)
+  end
+
+  #
+  # Returns current error state
+  #
+  def error
+    return @error if (@error)
+    return @current_data_service.error if @current_data_service
+    return 'none'
+  end
+
+  def is_local?
+    if (@current_data_service)
+      return (@current_data_service.name == 'local_db_service')
+    end
+
+    return false
+  end
+
+  #
+  # Determines if the data service is active
+  #
+  def active
+    if (@current_data_service)
+      return @current_data_service.active
+    end
+
+    return false
+  end
+
+  #
+  # Registers a data service with the proxy and immediately
+  # set as primary if online
+  #
+  def register_data_service(data_service, online=false)
+    validate(data_service)
+    data_service_id = @data_service_id += 1
+    @data_services[data_service_id] = data_service
+    set_data_service(data_service_id, online)
+  end
+
+  #
+  # Set the data service to be used
+  #
+  def set_data_service(data_service_id, online=false)
+    data_service = @data_services[data_service_id.to_i]
+    if (data_service.nil?)
+      raise "Data service with id: #{data_service_id} does not exist"
+    end
+
+    if (!online && !data_service.active)
+      raise "Data service not online: #{data_service.name}, not setting as active"
+    end
+
+    @current_data_service = data_service
+  end
+
+  #
+  # Retrieves metadata about the data services
+  #
+  def get_services_metadata()
+    services_metadata = []
+    @data_services.each_key {|key|
+      name = @data_services[key].name
+      active = !@current_data_service.nil? && name == @current_data_service.name
+      services_metadata << Metasploit::Framework::DataService::Metadata.new(key, name, active)
+    }
+
+    services_metadata
+  end
+
+  #
+  # Used to bridge the local db
+  #
+  def method_missing(method, *args, &block)
+    dlog ("Attempting to delegate method: #{method}")
+    unless @current_data_service.nil?
+      @current_data_service.send(method, *args, &block)
+    end
+  end
+
+  def respond_to?(method_name, include_private=false)
+    unless @current_data_service.nil?
+      return @current_data_service.respond_to?(method_name, include_private)
+    end
+
+    false
+  end
+
+  def get_data_service
+    raise 'No registered data_service' unless @current_data_service
+    return @current_data_service
+  end
+
+  def log_error(exception, ui_message)
+    elog "#{ui_message}: #{exception.message}"
+    exception.backtrace.each { |line| elog "#{line}" }
+    # TODO: We should try to surface the original exception, instead of just a generic one.
+    # This should not display the full backtrace, only the message.
+    raise Exception, "#{ui_message}: #{exception.message}. See log for more details."
+  end
+
+  #######
+  private
+  #######
+
+  def setup(opts)
+    begin
+      db_manager = opts.delete(:db_manager)
+      if !db_manager.nil?
+        register_data_service(db_manager, true)
+        @usable = true
+      else
+        @error = 'disabled'
+      end
+    rescue Exception => e
+      raise "Unable to initialize data service: #{e.message}"
+    end
+  end
+
+  def validate(data_service)
+    raise "Invalid data_service: #{data_service.class}, not of type Metasploit::Framework::DataService" unless data_service.is_a? (Metasploit::Framework::DataService)
+    raise 'Cannot register null data service data_service' unless data_service
+    raise 'Data Service already exists' if data_service_exist?(data_service)
+  end
+
+  def data_service_exist?(data_service)
+    @data_services.each_value{|value|
+      if (value.name == data_service.name)
+        return true
+      end
+    }
+
+    return false
+  end
+
+end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/credential_data_proxy.rb

@@ -0,0 +1,20 @@
+module CredentialDataProxy
+
+  def create_credential(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.create_credential(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem creating credential")
+    end
+  end
+
+  def creds(opts = {})
+    begin
+      data_service = self.get_data_service
+      data_service.creds(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem retrieving credentials")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/data_proxy_auto_loader.rb

@@ -0,0 +1,37 @@
+#
+# Autoloads specific data proxies
+#
+module DataProxyAutoLoader
+  autoload :HostDataProxy, 'metasploit/framework/data_service/proxy/host_data_proxy'
+  autoload :VulnDataProxy, 'metasploit/framework/data_service/proxy/vuln_data_proxy'
+  autoload :EventDataProxy, 'metasploit/framework/data_service/proxy/event_data_proxy'
+  autoload :WorkspaceDataProxy, 'metasploit/framework/data_service/proxy/workspace_data_proxy'
+  autoload :NoteDataProxy, 'metasploit/framework/data_service/proxy/note_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :ServiceDataProxy, 'metasploit/framework/data_service/proxy/service_data_proxy'
+  autoload :SessionDataProxy, 'metasploit/framework/data_service/proxy/session_data_proxy'
+  autoload :ExploitDataProxy, 'metasploit/framework/data_service/proxy/exploit_data_proxy'
+  autoload :LootDataProxy, 'metasploit/framework/data_service/proxy/loot_data_proxy'
+  autoload :SessionEventDataProxy, 'metasploit/framework/data_service/proxy/session_event_data_proxy'
+  autoload :CredentialDataProxy, 'metasploit/framework/data_service/proxy/credential_data_proxy'
+  autoload :NmapDataProxy, 'metasploit/framework/data_service/proxy/nmap_data_proxy'
+  autoload :DbExportDataProxy, 'metasploit/framework/data_service/proxy/db_export_data_proxy'
+  autoload :VulnAttemptDataProxy, 'metasploit/framework/data_service/proxy/vuln_attempt_data_proxy'
+
+  include ServiceDataProxy
+  include HostDataProxy
+  include VulnDataProxy
+  include EventDataProxy
+  include WorkspaceDataProxy
+  include NoteDataProxy
+  include WebDataProxy
+  include SessionDataProxy
+  include ExploitDataProxy
+  include LootDataProxy
+  include SessionEventDataProxy
+  include CredentialDataProxy
+  include NmapDataProxy
+  include DbExportDataProxy
+  include VulnAttemptDataProxy
+end

lib/metasploit/framework/data_service/proxy/db_export_data_proxy.rb

@@ -0,0 +1,14 @@
+module DbExportDataProxy
+  def run_db_export(path, format)
+    begin
+      data_service = self.get_data_service
+      opts = {
+          path: path,
+          format: format
+      }
+      data_service.run_db_export(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem generating DB Export")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/event_data_proxy.rb

@@ -0,0 +1,12 @@
+module EventDataProxy
+
+  def report_event(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_event(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting event")
+    end
+  end
+
+end

lib/metasploit/framework/data_service/proxy/exploit_data_proxy.rb

@@ -0,0 +1,29 @@
+module ExploitDataProxy
+
+  def report_exploit_attempt(host, opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_exploit_attempt(host, opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting exploit attempt")
+    end
+  end
+
+  def report_exploit_failure(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_exploit_failure(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting exploit failure")
+    end
+  end
+
+  def report_exploit_success(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_exploit_success(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting exploit success")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/host_data_proxy.rb

@@ -0,0 +1,78 @@
+module HostDataProxy
+
+  def hosts(wspace = workspace, non_dead = false, addresses = nil, search_term = nil)
+    begin
+      data_service = self.get_data_service()
+      opts = {}
+      opts[:wspace] = wspace
+      opts[:non_dead] = non_dead
+      opts[:address] = addresses
+      opts[:search_term] = search_term
+      data_service.hosts(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem retrieving hosts")
+    end
+  end
+
+  # TODO: Shouldn't this proxy to RemoteHostDataService#find_or_create_host ?
+  # It's currently skipping the "find" part
+  def find_or_create_host(opts)
+    report_host(opts)
+  end
+
+  def report_host(opts)
+    return unless valid(opts)
+
+    begin
+      data_service = self.get_data_service()
+      data_service.report_host(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting host")
+    end
+  end
+
+  def report_hosts(hosts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_hosts(hosts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting hosts")
+    end
+  end
+
+  def update_host(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.update_host(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem updating host")
+    end
+  end
+
+  def delete_host(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.delete_host(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem deleting host")
+    end
+  end
+
+  private
+
+  def valid(opts)
+    unless opts[:host]
+      ilog 'Invalid host hash passed, :host is missing'
+      return false
+    end
+
+    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
+    if opts[:host].eql? "Remote Pipe"
+      ilog "Invalid host hash passed, address was of type 'Remote Pipe'"
+      return false
+    end
+
+    return true
+  end
+
+end

lib/metasploit/framework/data_service/proxy/loot_data_proxy.rb

@@ -0,0 +1,41 @@
+module LootDataProxy
+
+  def report_loot(opts)
+    begin
+      data_service = self.get_data_service()
+      if !data_service.is_a?(Msf::DBManager)
+        opts[:data] = Base64.urlsafe_encode64(opts[:data]) if opts[:data]
+      end
+      data_service.report_loot(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting loot")
+    end
+  end
+
+  # TODO: Shouldn't this proxy to RemoteLootDataService#find_or_create_loot ?
+  # It's currently skipping the "find" part
+  def find_or_create_loot(opts)
+    report_loot(opts)
+  end
+
+  def loots(wspace, opts = {})
+    begin
+      data_service = self.get_data_service
+      opts[:wspace] = wspace
+      data_service.loot(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem retrieving loot")
+    end
+  end
+
+  alias_method :loot, :loots
+
+  def update_loot(opts)
+    begin
+      data_service = self.get_data_service
+      data_service.update_loot(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem updating loot")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/nmap_data_proxy.rb

@@ -0,0 +1,11 @@
+module NmapDataProxy
+
+  def import_nmap_xml_file(args = {})
+    begin
+      data_service = self.get_data_service()
+      data_service.import_nmap_xml_file(args)
+    rescue Exception => e
+      self.log_error(e, "Problem importing Nmap XML file")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/note_data_proxy.rb

@@ -0,0 +1,10 @@
+module NoteDataProxy
+  def report_note(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_note(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting note")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/service_data_proxy.rb

@@ -0,0 +1,43 @@
+module ServiceDataProxy
+
+  def services(wspace = workspace, opts = {})
+    begin
+      data_service = self.get_data_service()
+      opts[:workspace] = wspace
+      data_service.services(opts)
+    rescue Exception => e
+      self.log_error(e, 'Problem retrieving services')
+    end
+  end
+
+  def find_or_create_service(opts)
+    report_service(opts)
+  end
+
+  def report_service(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_service(opts)
+    rescue  Exception => e
+      self.log_error(e, 'Problem reporting service')
+    end
+  end
+
+  def update_service(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.update_service(opts)
+    rescue Exception => e
+      self.log_error(e, 'Problem updating service')
+    end
+  end
+
+  def delete_service(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.delete_service(opts)
+    rescue Exception => e
+      self.log_error(e, 'Problem deleting service')
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/session_data_proxy.rb

@@ -0,0 +1,14 @@
+module SessionDataProxy
+  def report_session(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_session(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting session")
+    end
+  end
+end
+
+
+
+

lib/metasploit/framework/data_service/proxy/session_event_data_proxy.rb

@@ -0,0 +1,11 @@
+module SessionEventDataProxy
+
+  def report_session_event(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_session_event(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting session event")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/vuln_attempt_data_proxy.rb

@@ -0,0 +1,20 @@
+module VulnAttemptDataProxy
+
+  def vuln_attempts(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.vuln_attempts(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem retrieving vulnerability attempts")
+    end
+  end
+
+  def report_vuln_attempt(vuln, opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_vuln_attempt(vuln, opts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting vulnerability attempts")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/vuln_data_proxy.rb

@@ -0,0 +1,39 @@
+
+module VulnDataProxy
+
+  def vulns(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.vulns(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem retrieving vulns")
+    end
+  end
+
+  def report_vuln(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_vuln(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem reporting vuln")
+    end
+  end
+
+  def update_vuln(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.update_vuln(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem updating vuln")
+    end
+  end
+
+  def delete_vuln(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.delete_vuln(opts)
+    rescue Exception => e
+      self.log_error(e, "Problem deleting vuln")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/web_data_proxy.rb

@@ -0,0 +1,10 @@
+module WebDataProxy
+  def report_web_site(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_web_site(opts)
+    rescue  Exception => e
+      self.log_error(e, "Problem reporting website")
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/workspace_data_proxy.rb

@@ -0,0 +1,66 @@
+module WorkspaceDataProxy
+
+  def find_workspace(workspace_name)
+    begin
+      data_service = self.get_data_service()
+      data_service.find_workspace(workspace_name)
+    rescue  Exception => e
+      self.log_error(e, "Problem finding workspace")
+    end
+  end
+
+  def add_workspace(workspace_name)
+    begin
+      data_service = self.get_data_service()
+      data_service.add_workspace(workspace_name)
+    rescue  Exception => e
+      self.log_error(e, "Problem adding workspace")
+    end
+  end
+
+  def default_workspace
+    begin
+      data_service = self.get_data_service()
+      data_service.default_workspace
+    rescue  Exception => e
+      self.log_error(e, "Problem finding default workspace")
+    end
+  end
+
+  def workspace
+    begin
+      data_service = self.get_data_service()
+      data_service.workspace
+    rescue  Exception => e
+      self.log_error(e, "Problem retrieving workspace")
+    end
+  end
+
+  def workspace=(workspace)
+    begin
+      data_service = self.get_data_service()
+      data_service.workspace = workspace
+    rescue  Exception => e
+      self.log_error(e, "Problem setting workspace")
+    end
+  end
+
+  def workspaces
+    begin
+      data_service = self.get_data_service()
+      data_service.workspaces
+    rescue  Exception => e
+      self.log_error(e, "Problem retrieving workspaces")
+    end
+  end
+
+  def workspace_associations_counts()
+    begin
+      data_service = self.get_data_service()
+      data_service.workspace_associations_counts()
+    rescue  Exception => e
+      self.log_error(e, "Problem retrieving workspace counts")
+    end
+  end
+
+end

lib/metasploit/framework/data_service/remote/http/core.rb

@@ -0,0 +1,303 @@
+require 'metasploit/framework/data_service'
+require 'metasploit/framework/data_service/remote/http/data_service_auto_loader'
+require 'net/http'
+require 'net/https'
+require 'uri'
+
+#
+# Parent data service for managing metasploit data in/on a separate process/machine over HTTP(s)
+#
+module Metasploit
+module Framework
+module DataService
+class RemoteHTTPDataService
+  include Metasploit::Framework::DataService
+  include DataServiceAutoLoader
+
+  ONLINE_TEST_URL = "/api/v1/online"
+  EXEC_ASYNC = { :exec_async => true }
+  GET_REQUEST = 'GET'
+  POST_REQUEST = 'POST'
+  DELETE_REQUEST = 'DELETE'
+  PUT_REQUEST = 'PUT'
+
+  #
+  # @param [String] endpoint A valid http or https URL. Cannot be nil
+  #
+  def initialize(endpoint, https_opts = {})
+    validate_endpoint(endpoint)
+    @endpoint = URI.parse(endpoint)
+    @https_opts = https_opts
+    build_client_pool(5)
+  end
+
+  def connection_established?
+    true
+  end
+
+  def after_establish_connection
+
+  end
+
+  def error
+    'none'
+  end
+
+  #
+  # POST data to the HTTP endpoint and don't wait for the endpoint to process the data before getting a response
+  #
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be posted. Cannot be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def post_data_async(path, data_hash, query = nil)
+    make_request(POST_REQUEST, path, data_hash.merge(EXEC_ASYNC), query)
+  end
+
+  #
+  # POST data to the HTTP endpoint
+  #
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be posted. Cannot be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def post_data(path, data_hash, query = nil)
+    make_request(POST_REQUEST, path, data_hash, query)
+  end
+
+  #
+  # GET data from the HTTP endpoint
+  #
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be included. Can be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def get_data(path, data_hash = nil, query = nil)
+    make_request(GET_REQUEST, path, data_hash, query)
+  end
+
+  #
+  # Send DELETE request to delete the specified resource from the HTTP endpoint
+  #
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be deleted. Cannot be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def delete_data(path, data_hash, query = nil)
+    make_request(DELETE_REQUEST, path, data_hash, query)
+  end
+
+  #
+  # Send PUT request to store data for the specified resource at the HTTP endpoint
+  #
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be stored. Cannot be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def put_data(path, data_hash, query = nil)
+    make_request(PUT_REQUEST, path, data_hash, query)
+  end
+
+  #
+  # Make the specified request_type
+  #
+  # @param request_type - A string representation of the HTTP method
+  # @param path - The URI path to send the request
+  # @param data_hash - A hash representation of the object to be included in the request. Cannot be nil or empty.
+  # @param query - A hash representation of the URI query data. Key-value pairs will be URL-encoded.
+  #
+  # @return A wrapped response (ResponseWrapper)
+  #
+  def make_request(request_type, path, data_hash = nil, query = nil)
+    begin
+      # simplify query by removing nil values
+      query_str = (!query.nil? && !query.empty?) ? append_workspace(query).compact.to_query : nil
+      uri = URI::HTTP::build({path: path, query: query_str})
+      dlog("HTTP #{request_type} request to #{uri.request_uri} with #{data_hash ? data_hash : "nil"}")
+
+      client = @client_pool.pop()
+      case request_type
+        when GET_REQUEST
+          request = Net::HTTP::Get.new(uri.request_uri)
+        when POST_REQUEST
+          request = Net::HTTP::Post.new(uri.request_uri)
+        when DELETE_REQUEST
+          request = Net::HTTP::Delete.new(uri.request_uri)
+        when PUT_REQUEST
+          request = Net::HTTP::Put.new(uri.request_uri)
+        else
+          raise Exception, 'A request_type must be specified'
+      end
+      built_request = build_request(request, data_hash)
+      response = client.request(built_request)
+
+      case response
+        when Net::HTTPOK
+          return SuccessResponse.new(response)
+        else
+          ilog "HTTP #{request_type} request: #{uri.request_uri} failed with code: #{response.code} message: #{response.body}"
+          return FailedResponse.new(response)
+      end
+    rescue EOFError => e
+      elog "No data was returned from the data service for request type/path : #{request_type}/#{path}, message: #{e.message}"
+      return FailedResponse.new('')
+    rescue Exception => e
+      elog "Problem with HTTP request for type/path: #{request_type}/#{path} message: #{e.message}"
+      return FailedResponse.new('')
+    ensure
+      @client_pool << client
+    end
+  end
+
+  #
+  # TODO: fix this
+  #
+  def active
+    return true
+  end
+
+  def name
+    "remote_data_service: (#{@endpoint})"
+  end
+
+  def set_header(key, value)
+    @headers = Hash.new() if @headers.nil?
+
+    @headers[key] = value
+  end
+
+  #########
+  protected
+  #########
+
+  #
+  # Simple response wrapper
+  #
+  class ResponseWrapper
+    attr_reader :response
+    attr_reader :expected
+
+    def initialize(response, expected)
+      @response = response
+      @expected = expected
+    end
+  end
+
+  #
+  # Failed response wrapper
+  #
+  class FailedResponse < ResponseWrapper
+    def initialize(response)
+      super(response, false)
+    end
+  end
+
+  #
+  # Success response wrapper
+  #
+  class SuccessResponse < ResponseWrapper
+    def initialize(response)
+      super(response, true)
+    end
+  end
+
+  #######
+  private
+  #######
+
+  def validate_endpoint(endpoint)
+    raise 'Endpoint cannot be nil' if endpoint.nil?
+  end
+
+  def append_workspace(data_hash)
+    workspace = data_hash[:workspace]
+    workspace = data_hash.delete(:wspace) unless workspace
+
+    if workspace && (workspace.is_a?(OpenStruct) || workspace.is_a?(::Mdm::Workspace))
+      data_hash[:workspace] = workspace.name
+    end
+
+    data_hash[:workspace] = current_workspace_name if workspace.nil?
+
+    data_hash
+  end
+
+  def build_request(request, data_hash)
+    request.content_type = 'application/json'
+    if !data_hash.nil? && !data_hash.empty?
+      data_hash.each do |k,v|
+        if v.is_a?(Msf::Session)
+          dlog('Dropping Msf::Session object before converting to JSON.')
+          dlog("data_hash is #{data_hash}")
+          dlog('Callstack:')
+          caller.each { |line| dlog("#{line}\n")}
+          data_hash.delete(k)
+        end
+      end
+      json_body = append_workspace(data_hash).to_json
+      request.body = json_body
+    end
+
+    if !@headers.nil? && !@headers.empty?
+      @headers.each do |key, value|
+        request[key] = value
+      end
+    end
+
+    request
+  end
+
+  def build_client_pool(size)
+    @client_pool = Queue.new()
+    (1..size).each {
+      http = Net::HTTP.new(@endpoint.host, @endpoint.port)
+      if @endpoint.is_a?(URI::HTTPS)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        unless @https_opts.empty?
+          if @https_opts[:skip_verify]
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          else
+            # https://stackoverflow.com/questions/22093042/implementing-https-certificate-pubkey-pinning-with-ruby
+            user_passed_cert = OpenSSL::X509::Certificate.new(File.read(@https_opts[:cert]))
+
+            http.verify_callback = lambda do |preverify_ok, cert_store|
+              server_cert = cert_store.chain[0]
+              return true unless server_cert.to_der == cert_store.current_cert.to_der
+              same_public_key?(server_cert, user_passed_cert)
+            end
+          end
+        end
+      end
+      @client_pool << http
+    }
+  end
+
+  # Tells us whether the private keys on the passed certificates match
+  # and use the same algo
+  def same_public_key?(ref_cert, actual_cert)
+    pkr, pka = ref_cert.public_key, actual_cert.public_key
+
+    # First check if the public keys use the same crypto...
+    return false unless pkr.class == pka.class
+    # ...and then - that they have the same contents
+    return false unless pkr.to_pem == pka.to_pem
+
+    true
+  end
+
+end
+end
+end
+end
+

lib/metasploit/framework/data_service/remote/http/data_service_auto_loader.rb

@@ -0,0 +1,36 @@
+#
+# Autoloads specific remote data services
+#
+module DataServiceAutoLoader
+  autoload :RemoteHostDataService, 'metasploit/framework/data_service/remote/http/remote_host_data_service'
+  autoload :RemoteEventDataService, 'metasploit/framework/data_service/remote/http/remote_event_data_service'
+  autoload :RemoteNoteDataService, 'metasploit/framework/data_service/remote/http/remote_note_data_service'
+  autoload :RemoteWorkspaceDataService, 'metasploit/framework/data_service/remote/http/remote_workspace_data_service'
+  autoload :RemoteVulnDataService, 'metasploit/framework/data_service/remote/http/remote_vuln_data_service'
+  autoload :RemoteWebDataService, 'metasploit/framework/data_service/remote/http/remote_web_data_service'
+  autoload :RemoteServiceDataService, 'metasploit/framework/data_service/remote/http/remote_service_data_service'
+  autoload :RemoteSessionDataService, 'metasploit/framework/data_service/remote/http/remote_session_data_service'
+  autoload :RemoteExploitDataService, 'metasploit/framework/data_service/remote/http/remote_exploit_data_service'
+  autoload :RemoteLootDataService, 'metasploit/framework/data_service/remote/http/remote_loot_data_service'
+  autoload :RemoteSessionEventDataService, 'metasploit/framework/data_service/remote/http/remote_session_event_data_service'
+  autoload :RemoteCredentialDataService, 'metasploit/framework/data_service/remote/http/remote_credential_data_service'
+  autoload :RemoteNmapDataService, 'metasploit/framework/data_service/remote/http/remote_nmap_data_service'
+  autoload :RemoteDbExportDataService, 'metasploit/framework/data_service/remote/http/remote_db_export_data_service'
+  autoload :RemoteVulnAttemptDataService, 'metasploit/framework/data_service/remote/http/remote_vuln_attempt_data_service'
+
+  include RemoteHostDataService
+  include RemoteEventDataService
+  include RemoteNoteDataService
+  include RemoteWorkspaceDataService
+  include RemoteVulnDataService
+  include RemoteWebDataService
+  include RemoteServiceDataService
+  include RemoteSessionDataService
+  include RemoteExploitDataService
+  include RemoteLootDataService
+  include RemoteSessionEventDataService
+  include RemoteCredentialDataService
+  include RemoteNmapDataService
+  include RemoteDbExportDataService
+  include RemoteVulnAttemptDataService
+end

lib/metasploit/framework/data_service/remote/http/query_meta.rb

@@ -0,0 +1,18 @@
+class QueryMeta
+  attr_accessor :filter_on
+  attr_accessor :associated_attributes
+
+  def initialize
+    @filter_on = []
+    @associated_attributes = []
+  end
+
+  def add_filter_item(item)
+    @filter_on << item
+  end
+
+  def add_associated_attribute(query_meta)
+    @associated_attributes << query_meta
+  end
+
+end

lib/metasploit/framework/data_service/remote/http/remote_credential_data_service.rb

@@ -0,0 +1,24 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteCredentialDataService
+  include ResponseDataHelper
+
+  CREDENTIAL_API_PATH = '/api/v1/credentials'
+  # "MDM_CLASS" is a little misleading since it is not in that repo but trying to keep naming consistent across DataServices
+  CREDENTIAL_MDM_CLASS = 'Metasploit::Credential::Core'
+
+  def creds(opts = {})
+    data = self.get_data(CREDENTIAL_API_PATH, opts)
+    rv = json_to_mdm_object(data, CREDENTIAL_MDM_CLASS, [])
+    parsed_body = JSON.parse(data.response.body)
+    parsed_body.each do |cred|
+      private_object = to_ar(cred['private_class'].constantize, cred['private'])
+      rv[parsed_body.index(cred)].private = private_object
+    end
+    rv
+  end
+
+  def create_credential(opts)
+    self.post_data_async(CREDENTIAL_API_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_db_export_data_service.rb

@@ -0,0 +1,13 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteDbExportDataService
+  include ResponseDataHelper
+
+  DB_EXPORT_API_PATH = '/api/v1/db-export'
+
+  def run_db_export(opts)
+    response = json_to_hash(self.get_data(DB_EXPORT_API_PATH, nil, opts))
+
+    process_file(response[:db_export_file], opts[:path])
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_event_data_service.rb

@@ -0,0 +1,7 @@
+module RemoteEventDataService
+  EVENT_API_PATH = '/api/v1/events'
+
+  def report_event(opts)
+    self.post_data_async(EVENT_API_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_exploit_data_service.rb

@@ -0,0 +1,20 @@
+module RemoteExploitDataService
+  EXPLOIT_API_PATH = '/api/v1/exploits'
+
+  def report_exploit_attempt(host, opts)
+    opts[:host] = host
+    opts[:exploit_report_type] = "attempt"
+    self.post_data_async(EXPLOIT_API_PATH, opts)
+  end
+
+  def report_exploit_failure(opts)
+    opts[:exploit_report_type] = "failure"
+    self.post_data_async(EXPLOIT_API_PATH, opts)
+  end
+
+  def report_exploit_success(opts)
+    opts[:exploit_report_type] = "success"
+    self.post_data_async(EXPLOIT_API_PATH, opts)
+  end
+
+end

lib/metasploit/framework/data_service/remote/http/remote_host_data_service.rb

@@ -0,0 +1,44 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteHostDataService
+  include ResponseDataHelper
+
+  HOST_API_PATH = '/api/v1/hosts'
+  HOST_SEARCH_PATH = HOST_API_PATH + "/search"
+  HOST_MDM_CLASS = 'Mdm::Host'
+
+  def hosts(opts)
+    json_to_mdm_object(self.get_data(HOST_API_PATH, nil, opts), HOST_MDM_CLASS, [])
+  end
+
+  def report_host(opts)
+    json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
+  end
+
+  def find_or_create_host(opts)
+    json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
+  end
+
+  def report_hosts(hosts)
+    self.post_data(HOST_API_PATH, hosts)
+  end
+
+  def update_host(opts)
+    path = HOST_API_PATH
+    if opts && opts[:id]
+      id = opts.delete(:id)
+      path = "#{HOST_API_PATH}/#{id}"
+    end
+    json_to_mdm_object(self.put_data(path, opts), HOST_MDM_CLASS, [])
+  end
+
+  def delete_host(opts)
+    json_to_mdm_object(self.delete_data(HOST_API_PATH, opts), HOST_MDM_CLASS, [])
+  end
+
+  # TODO: Remove? What is the purpose of this method?
+  def do_host_search(search)
+    response = self.post_data(HOST_SEARCH_PATH, search)
+    return response.body
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_loot_data_service.rb

@@ -0,0 +1,46 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteLootDataService
+  include ResponseDataHelper
+
+  LOOT_API_PATH = '/api/v1/loots'
+  LOOT_MDM_CLASS = 'Mdm::Loot'
+
+  def loot(opts = {})
+    # TODO: Add an option to toggle whether the file data is returned or not
+    loots = json_to_mdm_object(self.get_data(LOOT_API_PATH, nil, opts), LOOT_MDM_CLASS, [])
+    # Save a local copy of the file
+    loots.each do |loot|
+      if loot.data
+        local_path = File.join(Msf::Config.loot_directory, File.basename(loot.path))
+        loot.path = process_file(loot.data, local_path)
+      end
+    end
+    loots
+  end
+
+  def report_loot(opts)
+    self.post_data_async(LOOT_API_PATH, opts)
+  end
+
+  def find_or_create_loot(opts)
+    json_to_mdm_object(self.post_data(LOOT_API_PATH, opts), LOOT_MDM_CLASS, [])
+  end
+
+  def report_loots(loot)
+    self.post_data(LOOT_API_PATH, loot)
+  end
+
+  def update_loot(opts)
+    path = LOOT_API_PATH
+    if opts && opts[:id]
+      id = opts.delete(:id)
+      path = "#{LOOT_API_PATH}/#{id}"
+    end
+    json_to_mdm_object(self.put_data(path, opts), LOOT_MDM_CLASS, [])
+  end
+
+  def delete_loot(opts)
+    json_to_mdm_object(self.delete_data(LOOT_API_PATH, opts), LOOT_MDM_CLASS, [])
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_nmap_data_service.rb

@@ -0,0 +1,19 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteNmapDataService
+  include ResponseDataHelper
+
+  NMAP_PATH = '/api/v1/nmaps'
+
+  def import_nmap_xml_file(opts)
+    filename = opts[:filename]
+    data = ""
+    File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+
+    opts[:data] = Base64.urlsafe_encode64(data)
+
+    self.post_data(NMAP_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_note_data_service.rb

@@ -0,0 +1,11 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteNoteDataService
+  include ResponseDataHelper
+
+  NOTE_API_PATH = '/api/v1/notes'
+
+  def report_note(opts)
+    self.post_data_async(NOTE_API_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_service_data_service.rb

@@ -0,0 +1,25 @@
+module RemoteServiceDataService
+  SERVICE_API_PATH  = '/api/v1/services'
+  SERVICE_MDM_CLASS = 'Mdm::Service'
+
+  def services(opts)
+    json_to_mdm_object(self.get_data(SERVICE_API_PATH, nil, opts), SERVICE_MDM_CLASS, [])
+  end
+
+  def report_service(opts)
+    json_to_mdm_object(self.post_data(SERVICE_API_PATH, opts), SERVICE_MDM_CLASS).first
+  end
+
+  def update_service(opts)
+    path = SERVICE_API_PATH
+    if opts && opts[:id]
+      id = opts.delete(:id)
+      path = "#{SERVICE_API_PATH}/#{id}"
+    end
+    json_to_mdm_object(self.put_data(path, opts), SERVICE_MDM_CLASS)
+  end
+
+  def delete_service(opts)
+    json_to_mdm_object(self.delete_data(SERVICE_API_PATH, opts), SERVICE_MDM_CLASS)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_session_data_service.rb

@@ -0,0 +1,84 @@
+module RemoteSessionDataService
+
+  SESSION_API_PATH = '/api/v1/sessions'
+  SESSION_MDM_CLASS = 'Mdm::Session'
+
+  def report_session(opts)
+    session = opts[:session]
+    if (session.kind_of? Msf::Session)
+      opts = convert_msf_session_to_hash(session)
+      opts[:session_dto] = true
+    elsif (opts[:host])
+      opts[:host] = opts[:host].address
+    end
+
+    opts[:time_stamp] = Time.now.utc
+    sess_db = json_to_mdm_object(self.post_data(SESSION_API_PATH, opts), SESSION_MDM_CLASS, []).first
+    session.db_record = sess_db
+  end
+
+  #######
+  private
+  #######
+
+  # TODO: handle task info
+  def convert_msf_session_to_hash(msf_session)
+    hash = Hash.new()
+    hash[:host_data] = parse_host_opts(msf_session)
+    hash[:session_data] = parse_session_data(msf_session)
+
+    if (msf_session.via_exploit)
+      hash[:vuln_info] = parse_vuln_info(msf_session)
+    end
+
+    return hash
+  end
+
+  def parse_host_opts(msf_session)
+    hash = Hash.new()
+    hash[:host] = msf_session.session_host
+    hash[:arch] = msf_session.arch if msf_session.respond_to?(:arch) and msf_session.arch
+    hash[:workspace] = msf_session[:workspace] || msf_session.workspace
+    return hash
+  end
+
+  def parse_session_data(msf_session)
+    hash = Hash.new()
+    # TODO: what to do with this shiz
+    hash[:datastore] = msf_session.exploit_datastore.to_h
+    hash[:desc] = msf_session.info
+    hash[:local_id] = msf_session.sid
+    hash[:platform] = msf_session.session_type
+    hash[:port] = msf_session.session_port
+    hash[:stype] = msf_session.type
+    hash[:via_exploit] = msf_session.via_exploit
+    hash[:via_payload] = msf_session.via_payload
+    return hash
+  end
+
+  def parse_host_opts(msf_session)
+    hash = Hash.new()
+    hash[:host] = msf_session.session_host
+    hash[:arch] = msf_session.arch if msf_session.respond_to?(:arch) and msf_session.arch
+    hash[:workspace] = msf_session.workspace || msf_session[:workspace]
+    return hash
+  end
+
+  def parse_vuln_info(msf_session)
+    hash = Hash.new()
+    if msf_session.via_exploit == "exploit/multi/handler" and msf_session.exploit_datastore['ParentModule']
+      hash[:mod_fullname] = msf_session.exploit_datastore['ParentModule']
+    else
+      hash[:mod_fullname] = msf_session.via_exploit
+    end
+
+    hash[:remote_port] = msf_session.exploit_datastore["RPORT"]
+    hash[:username] = msf_session.username
+    hash[:run_id] = msf_session.exploit.user_data.try(:[], :run_id)
+
+    hash[:mod_name] = msf_session.exploit.name
+    hash[:mod_references] = msf_session.exploit.references
+    return hash
+  end
+end
+

lib/metasploit/framework/data_service/remote/http/remote_session_event_data_service.rb

@@ -0,0 +1,17 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteSessionEventDataService
+  include ResponseDataHelper
+
+  SESSION_EVENT_API_PATH = '/api/v1/session-events'
+  SESSION_EVENT_MDM_CLASS = 'Mdm::SessionEvent'
+
+  def session_events(opts = {})
+    json_to_mdm_object(self.get_data(SESSION_EVENT_API_PATH, opts), SESSION_EVENT_MDM_CLASS, [])
+  end
+
+  def report_session_event(opts)
+    opts[:session] = opts[:session].db_record
+    self.post_data_async(SESSION_EVENT_API_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_vuln_attempt_data_service.rb

@@ -0,0 +1,17 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteVulnAttemptDataService
+  include ResponseDataHelper
+
+  VULN_ATTEMPT_API_PATH = '/api/v1/vuln-attempts'
+  VULN_ATTEMPT_MDM_CLASS = 'Mdm::VulnAttempt'
+
+  def vuln_attempts(opts)
+    json_to_mdm_object(self.get_data(VULN_ATTEMPT_API_PATH, nil, opts), VULN_ATTEMPT_MDM_CLASS, [])
+  end
+
+  def report_vuln_attempt(vuln, opts)
+    opts[:vuln_id] = vuln.id
+    json_to_mdm_object(self.post_data(VULN_ATTEMPT_API_PATH, opts), VULN_ATTEMPT_MDM_CLASS, []).first
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_vuln_data_service.rb

@@ -0,0 +1,29 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteVulnDataService
+  include ResponseDataHelper
+
+  VULN_API_PATH = '/api/v1/vulns'
+  VULN_MDM_CLASS = 'Mdm::Vuln'
+
+  def vulns(opts)
+    json_to_mdm_object(self.get_data(VULN_API_PATH, nil, opts), VULN_MDM_CLASS, [])
+  end
+
+  def report_vuln(opts)
+    json_to_mdm_object(self.post_data(VULN_API_PATH, opts), VULN_MDM_CLASS, []).first
+  end
+
+  def update_vuln(opts)
+    path = VULN_API_PATH
+    if opts && opts[:id]
+      id = opts.delete(:id)
+      path = "#{VULN_API_PATH}/#{id}"
+    end
+    json_to_mdm_object(self.put_data(path, opts), VULN_MDM_CLASS, [])
+  end
+
+  def delete_vuln(opts)
+    json_to_mdm_object(self.delete_data(VULN_API_PATH, opts), VULN_MDM_CLASS, [])
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_web_data_service.rb

@@ -0,0 +1,11 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteWebDataService
+  include ResponseDataHelper
+
+  WEB_API_PATH = '/api/v1/webs'
+
+  def report_web_site(opts)
+    self.post_data_async(WEB_API_PATH, opts)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_workspace_data_service.rb

@@ -0,0 +1,57 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteWorkspaceDataService
+  include ResponseDataHelper
+
+  # TODO: should counts be a flag in query data for the workspaces resource?
+  WORKSPACE_COUNTS_API_PATH = '/api/v1/workspaces/counts'
+  WORKSPACE_API_PATH = '/api/v1/workspaces'
+  WORKSPACE_MDM_CLASS = 'Mdm::Workspace'
+  DEFAULT_WORKSPACE_NAME = 'default'
+
+  def find_workspace(workspace_name)
+    workspace = workspace_cache[workspace_name]
+    return workspace unless (workspace.nil?)
+
+    workspace = json_to_mdm_object(self.get_data(WORKSPACE_API_PATH, {:workspace_name => workspace_name}), WORKSPACE_MDM_CLASS).first
+    workspace_cache[workspace_name] = workspace
+  end
+
+  def add_workspace(workspace_name)
+    response = self.post_data(WORKSPACE_API_PATH, {:workspace_name => workspace_name})
+    json_to_mdm_object(response, WORKSPACE_MDM_CLASS, nil)
+  end
+
+  def default_workspace
+    find_workspace(DEFAULT_WORKSPACE_NAME)
+  end
+
+  def workspace
+    find_workspace(current_workspace_name)
+  end
+
+  def workspace=(workspace)
+    @current_workspace_name = workspace.name
+  end
+
+  def workspaces
+    json_to_mdm_object(self.get_data(WORKSPACE_API_PATH, {:all => true}), WORKSPACE_MDM_CLASS, [])
+  end
+
+  def workspace_associations_counts()
+    json_to_mdm_object(self.get_data(WORKSPACE_COUNTS_API_PATH, []), WORKSPACE_MDM_CLASS, [])
+  end
+
+  #########
+  protected
+  #########
+
+  def workspace_cache
+    @workspace_cache ||= {}
+  end
+
+  def current_workspace_name
+    @current_workspace_name ||= DEFAULT_WORKSPACE_NAME
+  end
+
+end

lib/metasploit/framework/data_service/remote/http/response_data_helper.rb

@@ -0,0 +1,159 @@
+require 'ostruct'
+require 'digest'
+
+#
+# HTTP response helper class
+#
+module ResponseDataHelper
+
+  #
+  # Converts an HTTP response to a Hash
+  #
+  # @param [ResponseWrapper] A wrapped HTTP response containing a JSON body.
+  # @return [Hash] A Hash interpretation of the JSON body.
+  #
+  def json_to_hash(response_wrapper)
+    begin
+      if response_wrapper.expected
+        body = response_wrapper.response.body
+        unless body.nil? && body.empty?
+          return JSON.parse(body).symbolize_keys
+        end
+      end
+    rescue Exception => e
+      elog "Error parsing response: #{e.message}"
+      e.backtrace.each { |line| elog line }
+    end
+  end
+
+  #
+  # Converts an HTTP response to an OpenStruct object
+  #
+  def json_to_open_struct_object(response_wrapper, returns_on_error = nil)
+    if response_wrapper.expected
+      begin
+        body = response_wrapper.response.body
+        if !body.nil? && !body.empty?
+          return JSON.parse(body, object_class: OpenStruct)
+        end
+      rescue Exception => e
+        elog "open struct conversion failed #{e.message}"
+      end
+    end
+
+    return returns_on_error
+  end
+
+  #
+  # Converts an HTTP response to an Mdm Object
+  #
+  # @param [ResponseWrapper] A wrapped HTTP response containing a JSON body.
+  # @param [String] The Mdm class to convert the JSON to.
+  # @param [Anything] A failsafe response to return if no objects are found.
+  # @return [ActiveRecord::Base] An object of type mdm_class, which inherits from ActiveRecord::Base
+  #
+  def json_to_mdm_object(response_wrapper, mdm_class, returns_on_error = nil)
+    if response_wrapper.expected
+      begin
+        body = response_wrapper.response.body
+        if !body.nil? && !body.empty?
+          parsed_body = Array.wrap(JSON.parse(body))
+          rv = []
+          parsed_body.each do |json_object|
+            rv << to_ar(mdm_class.constantize, json_object)
+          end
+          return rv
+        end
+      rescue Exception => e
+        elog "Mdm Object conversion failed #{e.message}"
+        e.backtrace.each { |line| elog "#{line}\n" }
+      end
+    end
+
+    return returns_on_error
+  end
+
+  # Processes a Base64 encoded file included in a JSON request.
+  # Saves the file in the location specified in the parameter.
+  #
+  # @param base64_file [String] The Base64 encoded file.
+  # @param save_path [String] The location to store the file. This should include the file's name.
+  # @return [String] The location where the file was successfully stored.
+  def process_file(base64_file, save_path)
+    decoded_file = Base64.urlsafe_decode64(base64_file)
+    begin
+      # If we are running the data service on the same box this will ensure we only write
+      # the file if it is somehow not there already.
+      unless File.exists?(save_path) && File.read(save_path) == decoded_file
+        File.open(save_path, 'w+') { |file| file.write(decoded_file) }
+      end
+    rescue Exception => e
+      elog "There was an error writing the file: #{e}"
+      e.backtrace.each { |line| elog "#{line}\n"}
+    end
+    save_path
+  end
+
+  # Converts a Hash or JSON string to an ActiveRecord object.
+  # Importantly, this retains associated objects if they are in the JSON string.
+  #
+  # Modified from https://github.com/swdyh/toar/
+  # Credit to https://github.com/swdyh
+  #
+  # @param [String] klass The ActiveRecord class to convert the JSON/Hash to.
+  # @param [String] val The JSON string, or Hash, to convert.
+  # @param [Class] base_class The base class to build back to. Used for recursion.
+  # @return [ActiveRecord::Base] A klass object, which inherits from ActiveRecord::Base.
+  def to_ar(klass, val, base_object = nil)
+    return nil unless val
+    data = val.class == Hash ? val.dup : JSON.parse(val)
+    obj = base_object || klass.new
+
+    obj_associations = klass.reflect_on_all_associations(:has_many).reduce({}) do |reflection, i|
+      reflection[i.options[:through]] = i if i.options[:through]
+      reflection
+    end
+
+    data.except(*obj.attributes.keys).each do |k, v|
+      association = klass.reflect_on_association(k)
+      next unless association
+
+      case association.macro
+        when :belongs_to
+          data.delete("#{k}_id")
+          to_ar(association.klass, v, obj.send("build_#{k}"))
+          obj.class_eval do
+            define_method("#{k}_id") { obj.send(k).id }
+          end
+        when :has_one
+          to_ar(association.klass, v, obj.send("build_#{k}"))
+        when :has_many
+          obj.send(k).proxy_association.target =
+              v.map { |i| to_ar(association.klass, i) }
+
+          as_th = obj_associations[k.to_sym]
+          if as_th
+            obj.send(as_th.name).proxy_association.target =
+                v.map { |i| to_ar(as_th.klass, i[as_th.source_reflection_name.to_s]) }
+          end
+      end
+    end
+    obj.assign_attributes(data.slice(*obj.attributes.keys))
+
+    obj.instance_eval do
+      # prevent save
+      def valid?(_context = nil)
+        false
+      end
+    end
+    obj
+  end
+
+  #
+  # Converts a hash to an open struct
+  #
+  def open_struct(hash)
+    OpenStruct.new(hash)
+  end
+
+end

lib/metasploit/framework/data_service/stubs/event_data_service.rb

@@ -0,0 +1,7 @@
+module EventDataService
+
+  def report_event(opts)
+    raise 'EventDataService#report_event is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/exploit_data_service.rb

@@ -0,0 +1,15 @@
+module ExploitDataService
+
+  def report_exploit_attempt(host, opts)
+    raise 'ExploitDataService#report_exploit_attempt is not implemented'
+  end
+
+  def report_exploit_failure(opts)
+    raise 'ExploitDataService#report_exploit_failure is not implemented'
+  end
+
+  def report_exploit_success(opts)
+    raise 'ExploitDataService#report_exploit_success is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/host_data_service.rb

@@ -0,0 +1,22 @@
+module HostDataService
+
+  def report_host(opts)
+    raise 'HostDataService#report_host is not implemented'
+  end
+
+  def report_hosts(hosts)
+    raise 'HostDataService#report_hosts is not implemented'
+  end
+
+  def hosts(opts)
+    raise 'HostDataService#hosts is not implemented'
+  end
+
+  def find_or_create_host(opts)
+    raise 'HostDataService#find_or_create_host is not implemented'
+  end
+
+  def delete_host(opts)
+    raise 'HostDataService#delete_host is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/loot_data_service.rb

@@ -0,0 +1,6 @@
+module LootDataService
+
+  def report_loot(opts)
+    raise 'LootDataService#report_loot is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/note_data_service.rb

@@ -0,0 +1,7 @@
+module NoteDataService
+
+  def report_note(opts)
+    raise 'NoteDataService#report_note is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/query_service.rb

@@ -0,0 +1,7 @@
+module QueryService
+
+  def do_nl_search(search)
+    raise 'QueryService#do_nl_search is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/search.rb

@@ -0,0 +1,4 @@
+class Search
+  attr_accessor :query
+  attr_accessor :projections
+end

lib/metasploit/framework/data_service/stubs/service_data_service.rb

@@ -0,0 +1,7 @@
+module ServiceDataService
+
+  def report_service(opts)
+    raise 'ServiceDataService#report_service is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/session_data_service.rb

@@ -0,0 +1,5 @@
+module SessionDataService
+  def report_session(opts)
+    raise 'SessionDataService#report_session is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/session_event_service.rb

@@ -0,0 +1,6 @@
+module SessionEventDataService
+
+  def report_session_event(opts)
+    raise 'SessionEventDataService#report_session_event is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/vuln_data_service.rb

@@ -0,0 +1,7 @@
+module VulnDataService
+
+  def report_vuln(opts)
+    raise 'VulnDataServicee#report_vuln is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/web_data_service.rb

@@ -0,0 +1,6 @@
+module WebDataService
+
+  def report_web_site(opts)
+    raise 'WebDataService#report_web_site is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/workspace_data_service.rb

@@ -0,0 +1,35 @@
+module WorkspaceDataService
+
+  def find_workspace(workspace_name)
+    raise 'WorkspaceDataService#find_workspace is not implemented'
+  end
+
+  def add_workspace(workspace_name)
+    raise 'WorkspaceDataService#add_workspace is not implemented'
+  end
+
+  def default_workspace
+    raise 'WorkspaceDataService#default_workspace is not implemented'
+  end
+
+  def workspace
+    raise 'WorkspaceDataService#workspace is not implemented'
+  end
+
+  def workspace=(workspace)
+    raise 'WorkspaceDataService#workspace= is not implemented'
+  end
+
+  def workspaces
+    raise 'WorkspaceDataService#workspaces is not implemented'
+  end
+
+  def workspace_associations_counts()
+    raise 'WorkspaceDataService#workspace_associations_counts is not implemented'
+  end
+
+  def rename_workspace(from_name, to_name)
+    raise 'WorkspaceDataService#rename_workspace is not implemented'
+  end
+
+end

lib/metasploit/framework/parsed_options/remote_db.rb

@@ -0,0 +1,46 @@
+require 'metasploit/framework/parsed_options/base'
+
+class Metasploit::Framework::ParsedOptions::RemoteDB < Metasploit::Framework::ParsedOptions::Base
+
+  def options
+    unless @options
+      super.tap { |options|
+        options.console = ActiveSupport::OrderedOptions.new
+
+        options.console.commands = []
+        options.console.confirm_exit = false
+        options.console.histfile = nil
+        options.console.local_output = nil
+        options.console.plugins = []
+        options.console.quiet = false
+        options.console.real_readline = false
+        options.console.resources = []
+        options.console.subcommand = :run
+      }
+    end
+
+    @options
+  end
+
+  #######
+  private
+  #######
+
+  def option_parser
+    unless @option_parser
+      super.tap { |option_parser|
+        option_parser.banner = "Usage: #{option_parser.program_name} [options]"
+
+        option_parser.separator ''
+        option_parser.separator 'Remote DB options:'
+
+        option_parser.on('-ns', "Remove signal processing") do
+          options.database.no_signal = true
+        end
+
+      }
+    end
+
+    @option_parser
+  end
+end

lib/msf/base/sessions/meterpreter.rb

@@ -501,14 +501,17 @@ class Meterpreter < Rex::Post::Meterpreter::Client
             end
           end
 
+          sysinfo = sys.config.sysinfo
+          host = Msf::Util::Host.normalize_host(self)
+
           framework.db.report_note({
             :type => "host.os.session_fingerprint",
-            :host => self,
+            :host => host,
             :workspace => wspace,
             :data => {
-              :name => sys.config.sysinfo["Computer"],
-              :os => sys.config.sysinfo["OS"],
-              :arch => sys.config.sysinfo["Architecture"],
+              :name => sysinfo["Computer"],
+              :os => sysinfo["OS"],
+              :arch => sysinfo["Architecture"],
             }
           })
 

lib/msf/core/auxiliary/report.rb

@@ -31,7 +31,8 @@ module Auxiliary::Report
 
   def create_credential(opts={})
     if active_db?
-      super(opts)
+      framework.db.create_credential(opts)
+      #super(opts)
     elsif !db_warning_given?
       vprint_warning('No active DB -- Credential data will not be saved!')
     end
@@ -293,7 +294,8 @@ module Auxiliary::Report
         :username     => username  || "unknown",
     }
 
-    vuln.vuln_attempts.create(attempt_info)
+    # TODO: figure out what opts are required and why the above logic doesn't match that of the db_manager method
+    framework.db.report_vuln_attempt(vuln, attempt_info)
 
     vuln
   end
@@ -389,7 +391,7 @@ module Auxiliary::Report
       ext = "txt"
     end
     # This method is available even if there is no database, don't bother checking
-    host = framework.db.normalize_host(host)
+    host = Msf::Util::Host.normalize_host(host)
 
     ws = (db ? myworkspace.name[0,16] : 'default')
     name =
@@ -416,6 +418,7 @@ module Auxiliary::Report
       conf[:workspace] = myworkspace
       conf[:name] = filename if filename
       conf[:info] = info if info
+      conf[:data] = data if data
 
       if service and service.kind_of?(::Mdm::Service)
         conf[:service] = service if service

lib/msf/core/db_export.rb

@@ -11,6 +11,9 @@ class Export
 
   attr_accessor :workspace
 
+  STATUS_START = "start"
+  STATUS_COMPLETE = "complete"
+
   def initialize(workspace)
     self.workspace = workspace
   end
@@ -31,20 +34,22 @@ class Export
 
   # Performs an export of the workspace's `Metasploit::Credential::Login` objects in pwdump format
   # @param path [String] the path on the local filesystem where the exported data will be written
-  # @return [void]
+  # @return [String] The path to the location of the written file.
   def to_pwdump_file(path, &block)
     exporter = Metasploit::Credential::Exporter::Pwdump.new(workspace: workspace)
 
-    File.open(path, 'w') do |file|
+    output_file = File.open(path, 'w') do |file|
       file << exporter.rendered_output
     end
-    true
+    output_file.path
   end
 
-
+  # Performs an export of the workspace's `Metasploit::Credential::Login` objects in XML format
+  # @param path [String] the path on the local filesystem where the exported data will be written
+  # @return [String] The path to the location of the written file.
   def to_xml_file(path, &block)
 
-    yield(:status, "start", "report") if block_given?
+    yield(:status, STATUS_START, "report") if block_given?
     extract_target_entries
     report_file = ::File.open(path, "wb")
 
@@ -52,49 +57,49 @@ class Export
     report_file.write %Q|<MetasploitV5>\n|
     report_file.write %Q|<generated time="#{Time.now.utc}" user="#{myusername}" project="#{myworkspace.name.gsub(/[^A-Za-z0-9\x20]/n,"_")}" product="framework"/>\n|
 
-    yield(:status, "start", "hosts") if block_given?
+    yield(:status, STATUS_START, "hosts") if block_given?
     report_file.write %Q|<hosts>\n|
     report_file.flush
     extract_host_info(report_file)
     report_file.write %Q|</hosts>\n|
 
-    yield(:status, "start", "events") if block_given?
+    yield(:status, STATUS_START, "events") if block_given?
     report_file.write %Q|<events>\n|
     report_file.flush
     extract_event_info(report_file)
     report_file.write %Q|</events>\n|
 
-    yield(:status, "start", "services") if block_given?
+    yield(:status, STATUS_START, "services") if block_given?
     report_file.write %Q|<services>\n|
     report_file.flush
     extract_service_info(report_file)
     report_file.write %Q|</services>\n|
 
-    yield(:status, "start", "web sites") if block_given?
+    yield(:status, STATUS_START, "web sites") if block_given?
     report_file.write %Q|<web_sites>\n|
     report_file.flush
     extract_web_site_info(report_file)
     report_file.write %Q|</web_sites>\n|
 
-    yield(:status, "start", "web pages") if block_given?
+    yield(:status, STATUS_START, "web pages") if block_given?
     report_file.write %Q|<web_pages>\n|
     report_file.flush
     extract_web_page_info(report_file)
     report_file.write %Q|</web_pages>\n|
 
-    yield(:status, "start", "web forms") if block_given?
+    yield(:status, STATUS_START, "web forms") if block_given?
     report_file.write %Q|<web_forms>\n|
     report_file.flush
     extract_web_form_info(report_file)
     report_file.write %Q|</web_forms>\n|
 
-    yield(:status, "start", "web vulns") if block_given?
+    yield(:status, STATUS_START, "web vulns") if block_given?
     report_file.write %Q|<web_vulns>\n|
     report_file.flush
     extract_web_vuln_info(report_file)
     report_file.write %Q|</web_vulns>\n|
 
-    yield(:status, "start", "module details") if block_given?
+    yield(:status, STATUS_START, "module details") if block_given?
     report_file.write %Q|<module_details>\n|
     report_file.flush
     extract_module_detail_info(report_file)
@@ -105,9 +110,9 @@ class Export
     report_file.flush
     report_file.close
 
-    yield(:status, "complete", "report") if block_given?
+    yield(:status, STATUS_COMPLETE, "report") if block_given?
 
-    true
+    report_file.path
   end
 
   # A convenience function that bundles together host, event, and service extraction.
@@ -543,4 +548,3 @@ class Export
 end
 end
 end
-

lib/msf/core/db_manager.rb

@@ -9,7 +9,6 @@ require 'rex/socket'
 #
 # Project
 #
-
 require 'metasploit/framework/require'
 require 'msf/base/config'
 require 'msf/core'
@@ -17,6 +16,8 @@ require 'msf/core/database_event'
 require 'msf/core/db_import_error'
 require 'msf/core/host_state'
 require 'msf/core/service_state'
+require 'metasploit/framework/data_service'
+
 
 # The db module provides persistent storage and events. This class should be instantiated LAST
 # as the active_suppport library overrides Kernel.require, slowing down all future code loads.
@@ -31,6 +32,7 @@ class Msf::DBManager
   autoload :Client, 'msf/core/db_manager/client'
   autoload :Connection, 'msf/core/db_manager/connection'
   autoload :Cred, 'msf/core/db_manager/cred'
+  autoload :DbExport, 'msf/core/db_manager/db_export'
   autoload :Event, 'msf/core/db_manager/event'
   autoload :ExploitAttempt, 'msf/core/db_manager/exploit_attempt'
   autoload :ExploitedHost, 'msf/core/db_manager/exploited_host'
@@ -60,10 +62,14 @@ class Msf::DBManager
 
   optionally_include_metasploit_credential_creation
 
+  # Interface must be included first
+  include Metasploit::Framework::DataService
+
   include Msf::DBManager::Adapter
   include Msf::DBManager::Client
   include Msf::DBManager::Connection
   include Msf::DBManager::Cred
+  include Msf::DBManager::DbExport
   include Msf::DBManager::Event
   include Msf::DBManager::ExploitAttempt
   include Msf::DBManager::ExploitedHost
@@ -93,6 +99,10 @@ class Msf::DBManager
   # Provides :framework and other accessors
   include Msf::Framework::Offspring
 
+  def name
+    'local_db_service'
+  end
+
   #
   # Attributes
   #
@@ -122,7 +132,7 @@ class Msf::DBManager
       return
     end
 
-    initialize_database_support
+    return initialize_database_support
   end
 
   #
@@ -163,4 +173,61 @@ class Msf::DBManager
 
     true
   end
+
+  def init_db(opts)
+
+    init_success = false
+
+    # Append any migration paths necessary to bring the database online
+    if opts['DatabaseMigrationPaths']
+      opts['DatabaseMigrationPaths'].each do |migrations_path|
+        ActiveRecord::Migrator.migrations_paths << migrations_path
+      end
+    end
+
+    if connection_established?
+      after_establish_connection
+    else
+      configuration_pathname = Metasploit::Framework::Database.configurations_pathname(path: opts['DatabaseYAML'])
+
+      unless configuration_pathname.nil?
+        if configuration_pathname.readable?
+          dbinfo = YAML.load_file(configuration_pathname) || {}
+          dbenv  = opts['DatabaseEnv'] || Rails.env
+          db     = dbinfo[dbenv]
+        else
+          elog("Warning, #{configuration_pathname} is not readable. Try running as root or chmod.")
+        end
+
+        if not db
+          elog("No database definition for environment #{dbenv}")
+        else
+          init_success = connect(db)
+        end
+      end
+    end
+
+    # framework.db.active will be true if after_establish_connection ran directly when connection_established? was
+    # already true or if framework.db.connect called after_establish_connection.
+    if !! error
+      if error.to_s =~ /RubyGem version.*pg.*0\.11/i
+        elog("***")
+        elog("*")
+        elog("* Metasploit now requires version 0.11 or higher of the 'pg' gem for database support")
+        elog("* There a three ways to accomplish this upgrade:")
+        elog("* 1. If you run Metasploit with your system ruby, simply upgrade the gem:")
+        elog("*    $ rvmsudo gem install pg ")
+        elog("* 2. Use the Community Edition web interface to apply a Software Update")
+        elog("* 3. Uninstall, download the latest version, and reinstall Metasploit")
+        elog("*")
+        elog("***")
+        elog("")
+        elog("")
+      end
+
+      elog("Failed to connect to the database: #{error}")
+    end
+
+    return init_success
+  end
 end

lib/msf/core/db_manager/connection.rb

@@ -18,7 +18,7 @@ module Msf::DBManager::Connection
       migrate
 
       # Set the default workspace
-      framework.db.workspace = framework.db.default_workspace
+      self.workspace = self.default_workspace
     rescue ::Exception => exception
       self.error = exception
       elog("DB.connect threw an exception: #{exception}")

lib/msf/core/db_manager/cred.rb

@@ -1,9 +1,41 @@
 module Msf::DBManager::Cred
   # This methods returns a list of all credentials in the database
-  def creds(wspace=workspace)
-  ::ActiveRecord::Base.connection_pool.with_connection {
-    Mdm::Cred.where("hosts.workspace_id = ?", wspace.id).joins(:service => :host)
-  }
+  def creds(opts)
+    query = nil
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      query = Metasploit::Credential::Core.where( workspace_id: framework.db.workspace.id )
+      query = query.includes(:private, :public, :logins).references(:private, :public, :logins)
+      query = query.includes(logins: [ :service, { service: :host } ])
+
+      if opts[:type].present?
+        query = query.where(metasploit_credential_privates: { type: opts[:type] })
+      end
+
+      if opts[:svcs].present?
+        query = query.where(Mdm::Service[:name].in(opts[:svcs]))
+      end
+
+      if opts[:ports].present?
+        query = query.where(Mdm::Service[:port].in(opts[:ports]))
+      end
+
+      if opts[:user].present?
+        # If we have a user regex, only include those that match
+        query = query.where('"metasploit_credential_publics"."username" ~* ?', opts[:user])
+      end
+
+      if opts[:pass].present?
+        # If we have a password regex, only include those that match
+        query = query.where('"metasploit_credential_privates"."data" ~* ?', opts[:pass])
+      end
+
+      if opts[:host_ranges] || opts[:ports] || opts[:svcs]
+        # Only find Cores that have non-zero Logins if the user specified a
+        # filter based on host, port, or service name
+        query = query.where(Metasploit::Credential::Login[:id].not_eq(nil))
+      end
+    }
+    query
   end
 
   # This method iterates the creds table calling the supplied block with the

lib/msf/core/db_manager/db_export.rb

@@ -0,0 +1,20 @@
+require 'msf/core/db_export'
+
+module Msf::DBManager::DbExport
+  def run_db_export(opts)
+    exporter = Msf::DBManager::Export.new(framework.db.workspace)
+
+    output_file = exporter.send("to_#{opts[:format]}_file".intern, opts[:path]) do |mtype, mstatus, mname|
+      if mtype == :status
+        if mstatus == Msf::DBManager::Export::STATUS_START
+          ilog "    >> Starting export of #{mname}"
+        end
+        if mstatus == Msf::DBManager::Export::STATUS_COMPLETE
+          ilog "    >> Finished export of #{mname}"
+        end
+      end
+    end
+
+    File.expand_path(output_file)
+  end
+end

lib/msf/core/db_manager/event.rb

@@ -8,7 +8,7 @@ module Msf::DBManager::Event
   def report_event(opts = {})
     return if not active
   ::ActiveRecord::Base.connection_pool.with_connection {
-    wspace = opts.delete(:workspace) || workspace
+    wspace = get_workspace(opts)
     return if not wspace # Temp fix?
     uname  = opts.delete(:username)
 

lib/msf/core/db_manager/host.rb

@@ -1,4 +1,5 @@
 module Msf::DBManager::Host
+  # TODO: doesn't appear to have any callers. How is this used?
   # Deletes a host and associated data matching this address/comm
   def del_host(wspace, address, comm='')
   ::ActiveRecord::Base.connection_pool.with_connection {
@@ -8,6 +9,29 @@ module Msf::DBManager::Host
   }
   end
 
+  # Deletes Host entries based on the IDs passed in.
+  #
+  # @param opts[:ids] [Array] Array containing Integers corresponding to the IDs of the Host entries to delete.
+  # @return [Array] Array containing the Mdm::Host objects that were successfully deleted.
+  def delete_host(opts)
+    raise ArgumentError.new("The following options are required: :ids") if opts[:ids].nil?
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      deleted = []
+      opts[:ids].each do |host_id|
+        host = Mdm::Host.find(host_id)
+        begin
+          deleted << host.destroy
+        rescue # refs suck
+          elog("Forcibly deleting #{host.address}")
+          deleted << host.delete
+        end
+      end
+
+      return deleted
+    }
+  end
+
   #
   # Iterates over the hosts table calling the supplied block with the host
   # instance of each entry.
@@ -22,9 +46,60 @@ module Msf::DBManager::Host
 
   # Exactly like report_host but waits for the database to create a host and returns it.
   def find_or_create_host(opts)
+    host = get_host(opts.clone)
+    return host unless host.nil?
+
     report_host(opts)
   end
 
+  def add_host_tag(opts)
+    workspace = opts[:workspace]
+    if workspace.kind_of? String
+      workspace = find_workspace(workspace)
+    end
+
+    ip = opts[:ip]
+    tag_name = opts[:tag_name]
+
+    host = framework.db.get_host(:workspace => workspace, :address => ip)
+    if host
+      possible_tags = Mdm::Tag.joins(:hosts).where("hosts.workspace_id = ? and hosts.address = ? and tags.name = ?", workspace.id, ip, tag_name).order("tags.id DESC").limit(1)
+      tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first)
+      tag.name = tag_name
+      tag.hosts = [host]
+      tag.save! if tag.changed?
+    end
+  end
+
+  def delete_host_tag(opts)
+    workspace = opts[:workspace]
+    if workspace.kind_of? String
+      workspace = find_workspace(workspace)
+    end
+
+    ip = opts[:rws]
+    tag_name = opts[:tag_name]
+
+    tag_ids = []
+    if ip.nil?
+      found_tags = Mdm::Tag.joins(:hosts).where("hosts.workspace_id = ? and tags.name = ?", workspace.id, tag_name)
+      found_tags.each do |t|
+        tag_ids << t.id
+      end
+    else
+      found_tags = Mdm::Tag.joins(:hosts).where("hosts.workspace_id = ? and hosts.address = ? and tags.name = ?", workspace.id, ip, tag_name)
+      found_tags.each do |t|
+        tag_ids << t.id
+      end
+    end
+
+    tag_ids.each do |id|
+      tag = Mdm::Tag.find_by_id(id)
+      tag.hosts.delete
+      tag.destroy
+    end
+  end
+
   #
   # Find a host.  Performs no database writes.
   #
@@ -43,7 +118,7 @@ module Msf::DBManager::Host
       wspace = find_workspace(wspace)
     end
 
-    address = normalize_host(address)
+    address = Msf::Util::Host.normalize_host(address)
     return wspace.hosts.find_by_address(address)
   }
   end
@@ -57,67 +132,28 @@ module Msf::DBManager::Host
   end
 
   # Returns a list of all hosts in the database
-  def hosts(wspace = workspace, only_up = false, addresses = nil)
-  ::ActiveRecord::Base.connection_pool.with_connection {
-    conditions = {}
-    conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up
-    conditions[:address] = addresses if addresses
-    wspace.hosts.where(conditions).order(:address)
-  }
-  end
+  def hosts(opts)
+    wspace = opts[:workspace] || opts[:wspace] || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
 
-  #
-  # Returns something suitable for the +:host+ parameter to the various report_* methods
-  #
-  # Takes a Host object, a Session object, an Msf::Session object or a String
-  # address
-  #
-  def normalize_host(host)
-    return host if defined?(::Mdm) && host.kind_of?(::Mdm::Host)
-    norm_host = nil
+    ::ActiveRecord::Base.connection_pool.with_connection {
 
-    if (host.kind_of? String)
+      conditions = {}
+      conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if opts[:non_dead]
+      conditions[:address] = opts[:address] if opts[:address] && !opts[:address].empty?
+      conditions[:id] = opts[:id] if opts[:id] && !opts[:id].empty?
 
-      if Rex::Socket.is_ipv4?(host)
-        # If it's an IPv4 addr with a port on the end, strip the port
-        if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/
-          norm_host = $1
-        else
-          norm_host = host
-        end
-      elsif Rex::Socket.is_ipv6?(host)
-        # If it's an IPv6 addr, drop the scope
-        address, scope = host.split('%', 2)
-        norm_host = address
+      if opts[:search_term] && !opts[:search_term].empty?
+        column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Host, opts[:search_term])
+        tag_conditions = Arel::Nodes::Regexp.new(Mdm::Tag.arel_table[:name], Arel::Nodes.build_quoted("(?mi)#{opts[:search_term]}"))
+        search_conditions = column_search_conditions.or(tag_conditions)
+        wspace.hosts.where(conditions).where(search_conditions).includes(:tags).references(:tags).order(:address)
       else
-        norm_host = Rex::Socket.getaddress(host, true)
+        wspace.hosts.where(conditions).order(:address)
       end
-    elsif defined?(::Mdm) && host.kind_of?(::Mdm::Session)
-      norm_host = host.host
-    elsif host.respond_to?(:session_host)
-      # Then it's an Msf::Session object
-      norm_host = host.session_host
-    end
-
-    # If we got here and don't have a norm_host yet, it could be a
-    # Msf::Session object with an empty or nil tunnel_host and tunnel_peer;
-    # see if it has a socket and use its peerhost if so.
-    if (
-        norm_host.nil? &&
-        host.respond_to?(:sock) &&
-        host.sock.respond_to?(:peerhost) &&
-        host.sock.peerhost.to_s.length > 0
-      )
-      norm_host = session.sock.peerhost
-    end
-    # If We got here and still don't have a real host, there's nothing left
-    # to try, just log it and return what we were given
-    if !norm_host
-      dlog("Host could not be normalized: #{host.inspect}")
-      norm_host = host
-    end
-
-    norm_host
+    }
   end
 
   def host_state_changed(host, ostate)
@@ -165,7 +201,7 @@ module Msf::DBManager::Host
     ret = { }
 
     if !addr.kind_of? ::Mdm::Host
-      addr = normalize_host(addr)
+      addr = Msf::Util::Host.normalize_host(addr)
 
       unless ipv46_validator(addr)
         raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
@@ -243,6 +279,20 @@ module Msf::DBManager::Host
   }
   end
 
+  def update_host(opts)
+    # process workspace string for update if included in opts
+    wspace = opts.delete(:workspace)
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+      opts[:workspace] = wspace
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      id = opts.delete(:id)
+      Mdm::Host.update(id, opts)
+    }
+  end
+
   def split_windows_os_name(os_name)
     return [] if os_name.nil?
     flavor_match = os_name.match(/Windows\s+(.*)/)
@@ -282,7 +332,7 @@ module Msf::DBManager::Host
     end
 
     if !addr.kind_of? ::Mdm::Host
-      addr = normalize_host(addr)
+      addr = Msf::Util::Host.normalize_host(addr)
       addr, scope = addr.split('%', 2)
       opts[:scope] = scope if scope
 

lib/msf/core/db_manager/http/db_manager_proxy.rb

@@ -0,0 +1,27 @@
+require 'singleton'
+require 'msf/core/db_manager'
+require 'msf/core/module_manager'
+require 'msf/core/constants'
+require 'metasploit/framework/database'
+require 'rails'
+
+class DBManagerProxy
+  include Singleton
+
+  attr_reader :db
+  attr_reader :modules
+
+  private
+
+  def initialize
+    @modules = Msf::ModuleManager.new(self, Msf::MODULE_TYPES)
+    @db = Msf::DBManager.new(self)
+    @db.init_db(parse_opts)
+  end
+
+  def parse_opts
+    opts = {}
+    opts['DatabaseYAML'] = Metasploit::Framework::Database.configurations_pathname.try(:to_path)
+    opts
+  end
+end

lib/msf/core/db_manager/http/http_db_manager_service.rb

@@ -0,0 +1,104 @@
+require 'rack'
+require 'msf/core/db_manager/http/sinatra_app'
+require 'metasploit/framework/parsed_options/remote_db'
+require 'rex/ui/text/output/stdio'
+
+class HttpDBManagerService
+
+  def start(opts)
+    parsed_options = Metasploit::Framework::ParsedOptions::RemoteDB.new
+    require_environment!(parsed_options)
+
+    if opts[:ssl]
+      ssl_opts = {}
+      ssl_opts[:private_key_file] = opts[:ssl_key]
+      ssl_opts[:cert_chain_file] = opts[:ssl_cert]
+      ssl_opts[:verify_peer] = false
+      opts[:ssl] = true
+      opts[:ssl_opts] = ssl_opts
+    end
+
+    init_db
+    start_http_server(opts)
+  end
+
+  private
+
+  def start_http_server(opts)
+
+    Rack::Handler::Thin.run(SinatraApp, opts) do |server|
+
+      if opts[:ssl] && opts[:ssl] = true
+        print_good "SSL Enabled"
+        server.ssl = true
+        server.ssl_options = opts[:ssl_opts]
+      else
+        print_warning 'SSL Disabled'
+      end
+      server.threaded = true
+    end
+  end
+
+  def init_db
+    DBManagerProxy.instance
+  end
+
+  def require_environment!(parsed_options)
+    # RAILS_ENV must be set before requiring 'config/application.rb'
+    parsed_options.environment!
+    ARGV.replace(parsed_options.positional)
+
+    # allow other Rails::Applications to use this command
+    if !defined?(Rails) || Rails.application.nil?
+      # @see https://github.com/rails/rails/blob/v3.2.17/railties/lib/rails/commands.rb#L39-L40
+      require Pathname.new(__FILE__).parent.parent.parent.parent.parent.parent.join('config', 'application')
+    end
+
+    # have to configure before requiring environment because
+    # config/environment.rb calls initialize! and the initializers will use
+    # the configuration from the parsed options.
+    parsed_options.configure(Rails.application)
+
+    Rails.application.require_environment!
+  end
+
+  # def init_servlets(http_server)
+  #   servlet_path = File.dirname(__FILE__) + '/servlet/*'
+  #   Dir.glob(servlet_path).collect{|file_path|
+  #     servlet_class = File.basename(file_path, '.rb').classify
+  #     require file_path
+  #     servlet_class_constant = servlet_class.constantize
+  #     http_server.mount servlet_class_constant.api_path, servlet_class_constant
+  #   }
+  # end
+
+
+
+end
+
+
+def print_line(msg)
+  $console_printer.print_line(msg)
+end
+
+def print_warning(msg)
+  $console_printer.print_warning(msg)
+end
+
+def print_good(msg)
+  $console_printer.print_good(msg)
+end
+
+def print_error(msg, exception = nil)
+  unless exception.nil?
+    msg += "\n    Call Stack:"
+    exception.backtrace.each {|line|
+      msg += "\n"
+      msg += "\t #{line}"
+    }
+  end
+
+  $console_printer.print_error(msg)
+end
+
+$console_printer = Rex::Ui::Text::Output::Stdio.new

lib/msf/core/db_manager/http/job_processor.rb

@@ -0,0 +1,39 @@
+require 'singleton'
+
+class JobProcessor
+  include Singleton
+
+  def submit_job(job_args, &job)
+    @job_queue << JobWraper.new(job_args, &job)
+  end
+
+  private
+
+  def initialize
+    @job_queue = Queue.new()
+    start_processor_thread()
+  end
+
+  def start_processor_thread()
+    Thread.new {
+      loop do
+        wrapper = @job_queue.pop()
+        begin
+          wrapper.job.call(wrapper.job_args)
+        rescue Exception => e
+          print_error "Error executing job #{e.message}", e
+        end
+      end
+    }
+  end
+
+  class JobWraper
+    attr_reader :job
+    attr_reader :job_args
+
+    def initialize(job_args, &job)
+      @job_args = job_args
+      @job = job
+    end
+  end
+end

lib/msf/core/db_manager/http/servlet/credential_servlet.rb

@@ -0,0 +1,46 @@
+module CredentialServlet
+
+  def self.api_path
+    '/api/v1/credentials'
+  end
+
+  def self.registered(app)
+    app.get CredentialServlet.api_path, &get_credentials
+    app.post CredentialServlet.api_path, &create_credential
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_credentials
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().creds(opts)
+        includes = [:logins, :public, :private, :realm]
+        # Need to append the human attribute into the private sub-object before converting to json
+        # This is normally pulled from a class method from the MetasploitCredential class
+        response = []
+        data.each do |cred|
+          json = cred.as_json(include: includes).merge('private_class' => cred.private.class.to_s)
+          response << json
+        end
+        set_json_response(response)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.create_credential
+    lambda {
+      job = lambda { |opts|
+        opts[:origin_type] = opts[:origin_type].to_sym
+        opts[:private_type] = opts[:private_type].to_sym
+        get_db().create_credential(opts)
+      }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/db_export_servlet.rb

@@ -0,0 +1,35 @@
+module DbExportServlet
+
+  def self.api_path
+    '/api/v1/db-export'
+  end
+
+  def self.registered(app)
+    app.get DbExportServlet.api_path, &get_db_export
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_db_export
+    lambda {
+      begin
+        opts = params.symbolize_keys
+	      opts[:path] = File.join(Msf::Config.local_directory, "#{File.basename(opts[:path])}-#{SecureRandom.hex}")
+
+        output_file = get_db.run_db_export(opts)
+
+        encoded_file = Base64.urlsafe_encode64(File.read(File.expand_path(output_file)))
+        response = {}
+        response[:db_export_file] = encoded_file
+        set_json_response(response)
+      rescue Exception => e
+        set_error_on_response(e)
+      ensure
+        # Ensure the temporary file gets cleaned up
+        File.delete(opts[:path])
+      end
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/event_servlet.rb

@@ -0,0 +1,21 @@
+module EventServlet
+
+  def self.api_path
+    '/api/v1/events'
+  end
+
+  def self.registered(app)
+    app.post EventServlet.api_path, &report_event
+  end
+
+  #######
+  private
+  #######
+
+  def self.report_event
+    lambda {
+      job = lambda { |opts| get_db().report_event(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/exploit_servlet.rb

@@ -0,0 +1,31 @@
+module ExploitServlet
+
+  def self.api_path
+    '/api/v1/exploits'
+  end
+
+  def self.registered(app)
+    app.post ExploitServlet.api_path, &report_exploit
+  end
+
+  #######
+  private
+  #######
+
+  def self.report_exploit
+    lambda {
+      job = lambda { |opts|
+        case opts[:exploit_report_type]
+          when "attempt"
+            get_db().report_exploit_attempt(opts[:host], opts)
+          when "failure"
+            get_db().report_exploit_failure(opts)
+          when "success"
+            get_db().report_exploit_success(opts)
+        end
+        get_db().report_host(opts)
+      }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/host_servlet.rb

@@ -0,0 +1,74 @@
+module HostServlet
+
+  def self.api_path
+    '/api/v1/hosts'
+  end
+
+  def self.api_path_with_id
+    "#{HostServlet.api_path}/?:id?"
+  end
+
+  def self.registered(app)
+    app.get HostServlet.api_path_with_id, &get_host
+    app.post HostServlet.api_path, &report_host
+    app.put HostServlet.api_path_with_id, &update_host
+    app.delete HostServlet.api_path, &delete_host
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_host
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().hosts(params.symbolize_keys)
+        includes = [:loots]
+        set_json_response(data, includes)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_host
+    lambda {
+      begin
+        job = lambda { |opts|
+          data = get_db().report_host(opts)
+        }
+        exec_report_job(request, &job)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.update_host
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        tmp_params = params.symbolize_keys
+        opts[:id] = tmp_params[:id] if tmp_params[:id]
+        data = get_db().update_host(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.delete_host
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().delete_host(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/loot_servlet.rb

@@ -0,0 +1,79 @@
+module LootServlet
+
+  def self.api_path
+    '/api/v1/loots'
+  end
+
+  def self.api_path_with_id
+    "#{LootServlet.api_path}/?:id?"
+  end
+
+  def self.registered(app)
+    app.get LootServlet.api_path, &get_loot
+    app.post LootServlet.api_path, &report_loot
+    app.put LootServlet.api_path_with_id, &update_loot
+    app.delete LootServlet.api_path, &delete_loot
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_loot
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().loots(params.symbolize_keys)
+        includes = [:host]
+        data.each do |loot|
+          loot.data = Base64.urlsafe_encode64(loot.data) if loot.data
+        end
+        set_json_response(data, includes)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_loot
+    lambda {
+      job = lambda { |opts|
+        if opts[:data]
+          filename = File.basename(opts[:path])
+          local_path = File.join(Msf::Config.loot_directory, filename)
+          opts[:path] = process_file(opts[:data], local_path)
+          opts[:data] = Base64.urlsafe_decode64(opts[:data])
+        end
+
+        get_db().report_loot(opts)
+      }
+      exec_report_job(request, &job)
+    }
+  end
+
+  def self.update_loot
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        tmp_params = params.symbolize_keys
+        opts[:id] = tmp_params[:id] if tmp_params[:id]
+        data = get_db().update_loot(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.delete_loot
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().delete_loot(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/nmap_servlet.rb

@@ -0,0 +1,27 @@
+module NmapServlet
+
+  def self.api_path
+    '/api/v1/nmaps'
+  end
+
+  def self.registered(app)
+    app.post NmapServlet.api_path, &import_nmap_xml_file
+  end
+
+  #######
+  private
+  #######
+
+  def self.import_nmap_xml_file
+    lambda {
+
+      job = lambda { |opts|
+        nmap_file = File.basename(opts[:filename])
+        nmap_file_path = File.join(Msf::Config.local_directory, nmap_file)
+        opts[:filename] = process_file(opts[:data], nmap_file_path)
+        get_db().import_nmap_xml_file(opts)
+      }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/note_servlet.rb

@@ -0,0 +1,22 @@
+module NoteServlet
+
+  def self.api_path
+    '/api/v1/notes'
+  end
+
+  def self.registered(app)
+    app.post NoteServlet.api_path, &report_note
+  end
+
+  #######
+  private
+  #######
+
+  def self.report_note
+    lambda {
+        job = lambda { |opts|  get_db().report_note(opts) }
+        exec_report_job(request, &job)
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/online_test_servlet.rb

@@ -0,0 +1,21 @@
+module OnlineTestServlet
+
+  def self.api_path
+    '/api/v1/online'
+  end
+
+  def self.registered(app)
+    app.get OnlineTestServlet.api_path, &get_active
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_active
+    lambda {
+      set_empty_response()
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/service_servlet.rb

@@ -0,0 +1,68 @@
+module ServiceServlet
+
+  def self.api_path
+    '/api/v1/services'
+  end
+
+  def self.api_path_with_id
+    "#{ServiceServlet.api_path}/?:id?"
+  end
+
+  def self.registered(app)
+    app.get  ServiceServlet.api_path, &get_services
+    app.post ServiceServlet.api_path, &report_service
+    app.put ServiceServlet.api_path_with_id, &update_service
+    app.delete ServiceServlet.api_path, &delete_service
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_services
+    lambda {
+      begin
+        opts = params.symbolize_keys
+        data = get_db.services(opts)
+        includes = [:host]
+        set_json_response(data, includes)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_service
+    lambda {
+      job = lambda { |opts| get_db.report_service(opts) }
+      includes = [:host]
+      exec_report_job(request, includes, &job)
+    }
+  end
+
+  def self.update_service
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        tmp_params = params.symbolize_keys
+        opts[:id] = tmp_params[:id] if tmp_params[:id]
+        data = get_db.update_service(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.delete_service
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db.delete_service(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/session_event_servlet.rb

@@ -0,0 +1,36 @@
+module SessionEventServlet
+
+  def self.api_path
+    '/api/v1/session-events'
+  end
+
+  def self.registered(app)
+    app.get SessionEventServlet.api_path, &get_session_event
+    app.post SessionEventServlet.api_path, &report_session_event
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_session_event
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().session_events(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_session_event
+    lambda {
+      job = lambda { |opts|
+        opts[:session] = open_struct(opts[:session][:table])
+        get_db().report_session_event(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/session_servlet.rb

@@ -0,0 +1,39 @@
+module SessionServlet
+  def self.api_path
+    '/api/v1/sessions'
+  end
+
+  def self.registered(app)
+    app.post SessionServlet.api_path, &report_session
+    app.get SessionServlet.api_path, &get_session
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_session
+    lambda {
+      begin
+        #opts = parse_json_request(request, false)
+        data = get_db().get_all_sessions()
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_session
+    lambda {
+      job = lambda { |opts|
+        if (opts[:session_data])
+          get_db().report_session_dto(opts)
+        else
+          get_db().report_session_host_dto(opts)
+        end
+      }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/vuln_attempt_servlet.rb

@@ -0,0 +1,46 @@
+module VulnAttemptServlet
+
+  def self.api_path
+    '/api/v1/vuln-attempts'
+  end
+
+  def self.api_path_with_id
+    "#{VulnAttemptServlet.api_path}/?:id?"
+  end
+
+  def self.registered(app)
+    app.get VulnAttemptServlet.api_path_with_id, &get_vuln_attempt
+    app.post VulnAttemptServlet.api_path, &report_vuln_attempt
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_vuln_attempt
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db.vuln_attempts(params.symbolize_keys)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_vuln_attempt
+    lambda {
+      begin
+        job = lambda { |opts|
+          vuln_id = opts.delete(:vuln_id)
+          vuln = get_db.vulns(id: vuln_id).first
+          get_db.report_vuln_attempt(vuln, opts)
+        }
+        exec_report_job(request, &job)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/vuln_servlet.rb

@@ -0,0 +1,74 @@
+module VulnServlet
+
+  def self.api_path
+    '/api/v1/vulns'
+  end
+
+  def self.api_path_with_id
+    "#{VulnServlet.api_path}/?:id?"
+  end
+
+  def self.registered(app)
+    app.get VulnServlet.api_path_with_id, &get_vuln
+    app.post VulnServlet.api_path, &report_vuln
+    app.put VulnServlet.api_path_with_id, &update_vuln
+    app.delete VulnServlet.api_path, &delete_vuln
+  end
+
+  #######
+  private
+  #######
+
+  def self.get_vuln
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db.vulns(params.symbolize_keys)
+        includes = [:host, :vulns_refs, :refs, :module_refs]
+        set_json_response(data, includes)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_vuln
+    lambda {
+      begin
+        job = lambda { |opts|
+          get_db.report_vuln(opts)
+        }
+        exec_report_job(request, &job)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.update_vuln
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        tmp_params = params.symbolize_keys
+        opts[:id] = tmp_params[:id] if tmp_params[:id]
+        data = get_db.update_vuln(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.delete_vuln
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db.delete_vuln(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/web_servlet.rb

@@ -0,0 +1,22 @@
+module WebServlet
+
+  def self.api_path
+    '/api/v1/webs'
+  end
+
+  def self.registered(app)
+    app.post WebServlet.api_path, &report_web
+  end
+
+  #######
+  private
+  #######
+
+  def self.report_web
+    lambda {
+      job = lambda { |opts|  get_db().report_web_site(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/workspace_servlet.rb

@@ -0,0 +1,57 @@
+module WorkspaceServlet
+
+    def self.api_path
+      '/api/v1/workspaces'
+    end
+
+    def self.registered(app)
+      app.get WorkspaceServlet.api_path, &get_workspace
+      app.get WorkspaceServlet.api_path + '/counts', &get_workspace_counts
+      app.post WorkspaceServlet.api_path, &add_workspace
+    end
+
+    #######
+    private
+    #######
+
+    def self.get_workspace
+      lambda {
+        begin
+          opts = parse_json_request(request, true)
+          includes = nil
+          if (opts[:all])
+            data = get_db().workspaces
+            #includes = 'hosts: {only: :count}, services: {only: :count}, vulns: {only: :count}, creds: {only: :count}, loots: {only: :count}, notes: {only: :count}'
+          else
+            data = get_db().find_workspace(opts[:workspace_name])
+          end
+
+          set_json_response(data, includes)
+        rescue Exception => e
+          set_error_on_response(e)
+        end
+      }
+    end
+
+    def self.get_workspace_counts
+      lambda {
+        begin
+          set_json_response(get_db().workspace_associations_counts)
+        rescue Exception => e
+          set_error_on_response(e)
+        end
+      }
+    end
+
+    def self.add_workspace
+      lambda {
+        begin
+          opts = parse_json_request(request, true)
+          workspace = get_db().add_workspace(opts[:workspace_name])
+          set_json_response(workspace)
+        rescue Exception => e
+          set_error_on_response(e)
+        end
+      }
+    end
+end

lib/msf/core/db_manager/http/servlet_helper.rb

@@ -0,0 +1,143 @@
+require 'json'
+require 'msf/core/db_manager/http/db_manager_proxy'
+require 'msf/core/db_manager/http/job_processor'
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module ServletHelper
+  include ResponseDataHelper
+
+  def set_error_on_response(error)
+    print_error "Error handling request: #{error.message}", error
+    headers = {'Content-Type' => 'text/plain'}
+    [500, headers, error.message]
+  end
+
+  def set_empty_response()
+    [200,  '']
+  end
+
+  def set_json_response(data, includes = nil)
+    headers = {'Content-Type' => 'application/json'}
+    [200, headers, to_json(data, includes)]
+  end
+
+  def parse_json_request(request, strict = false)
+    body = request.body.read
+    if (body.nil? || body.empty?)
+      raise 'Invalid body, expected data' if strict
+      return {}
+    end
+
+    hash = JSON.parse(body)
+    hash.deep_symbolize_keys
+  end
+
+  def exec_report_job(request, includes = nil, &job)
+    begin
+
+      # report jobs always need data
+      opts = parse_json_request(request, true)
+
+      exec_async = opts.delete(:exec_async)
+      if (exec_async)
+        JobProcessor.instance.submit_job(opts, &job)
+        return set_empty_response()
+      else
+        data = job.call(opts)
+        return set_json_response(data, includes)
+      end
+
+    rescue Exception => e
+      set_error_on_response(e)
+    end
+  end
+
+  def get_db()
+    DBManagerProxy.instance.db
+  end
+
+  #######
+  private
+  #######
+
+  def to_json(data, includes = nil)
+    return '{}' if data.nil?
+    json = includes.nil? ? data.to_json : data.to_json(include:  includes)
+    return json.to_s
+  end
+
+
+  # TODO: add query meta
+  # Returns a hash representing the model. Some configuration can be
+  # passed through +options+.
+  #
+  # The option <tt>include_root_in_json</tt> controls the top-level behavior
+  # of +as_json+. If +true+, +as_json+ will emit a single root node named
+  # after the object's type. The default value for <tt>include_root_in_json</tt>
+  # option is +false+.
+  #
+  #   user = User.find(1)
+  #   user.as_json
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #     "created_at" => "2006/08/01", "awesome" => true}
+  #
+  #   ActiveRecord::Base.include_root_in_json = true
+  #
+  #   user.as_json
+  #   # => { "user" => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #                  "created_at" => "2006/08/01", "awesome" => true } }
+  #
+  # This behavior can also be achieved by setting the <tt>:root</tt> option
+  # to +true+ as in:
+  #
+  #   user = User.find(1)
+  #   user.as_json(root: true)
+  #   # => { "user" => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #                  "created_at" => "2006/08/01", "awesome" => true } }
+  #
+  # Without any +options+, the returned Hash will include all the model's
+  # attributes.
+  #
+  #   user = User.find(1)
+  #   user.as_json
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true}
+  #
+  # The <tt>:only</tt> and <tt>:except</tt> options can be used to limit
+  # the attributes included, and work similar to the +attributes+ method.
+  #
+  #   user.as_json(only: [:id, :name])
+  #   # => { "id" => 1, "name" => "Konata Izumi" }
+  #
+  #   user.as_json(except: [:id, :created_at, :age])
+  #   # => { "name" => "Konata Izumi", "awesome" => true }
+  #
+  # To include the result of some method calls on the model use <tt>:methods</tt>:
+  #
+  #   user.as_json(methods: :permalink)
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "permalink" => "1-konata-izumi" }
+  #
+  # To include associations use <tt>:include</tt>:
+  #
+  #   user.as_json(include: :posts)
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "posts" => [ { "id" => 1, "author_id" => 1, "title" => "Welcome to the weblog" },
+  #   #                   { "id" => 2, "author_id" => 1, "title" => "So I was thinking" } ] }
+  #
+  # Second level and higher order associations work as well:
+  #
+  #   user.as_json(include: { posts: {
+  #                              include: { comments: {
+  #                                             only: :body } },
+  #                              only: :title } })
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "posts" => [ { "comments" => [ { "body" => "1st post!" }, { "body" => "Second!" } ],
+  #   #                     "title" => "Welcome to the weblog" },
+  #   #                   { "comments" => [ { "body" => "Don't think too hard" } ],
+  #   #
+
+end

lib/msf/core/db_manager/http/sinatra_app.rb

@@ -0,0 +1,41 @@
+require 'sinatra/base'
+require 'msf/core/db_manager/http/servlet_helper'
+require 'msf/core/db_manager/http/servlet/host_servlet'
+require 'msf/core/db_manager/http/servlet/note_servlet'
+require 'msf/core/db_manager/http/servlet/vuln_servlet'
+require 'msf/core/db_manager/http/servlet/event_servlet'
+require 'msf/core/db_manager/http/servlet/web_servlet'
+require 'msf/core/db_manager/http/servlet/online_test_servlet'
+require 'msf/core/db_manager/http/servlet/workspace_servlet'
+require 'msf/core/db_manager/http/servlet/service_servlet'
+require 'msf/core/db_manager/http/servlet/session_servlet'
+require 'msf/core/db_manager/http/servlet/exploit_servlet'
+require 'msf/core/db_manager/http/servlet/loot_servlet'
+require 'msf/core/db_manager/http/servlet/session_event_servlet'
+require 'msf/core/db_manager/http/servlet/credential_servlet'
+require 'msf/core/db_manager/http/servlet/nmap_servlet'
+require 'msf/core/db_manager/http/servlet/db_export_servlet'
+require 'msf/core/db_manager/http/servlet/vuln_attempt_servlet'
+
+class SinatraApp < Sinatra::Base
+
+  helpers ServletHelper
+
+  # Servlet registration
+  register HostServlet
+  register VulnServlet
+  register EventServlet
+  register WebServlet
+  register OnlineTestServlet
+  register NoteServlet
+  register WorkspaceServlet
+  register ServiceServlet
+  register SessionServlet
+  register ExploitServlet
+  register LootServlet
+  register SessionEventServlet
+  register CredentialServlet
+  register NmapServlet
+  register DbExportServlet
+  register VulnAttemptServlet
+end

lib/msf/core/db_manager/loot.rb

@@ -24,16 +24,32 @@ module Msf::DBManager::Loot
   #
   # This methods returns a list of all loot in the database
   #
-  def loots(wspace=workspace)
-  ::ActiveRecord::Base.connection_pool.with_connection {
-    wspace.loots
-  }
+  def loots(opts)
+    wspace = opts.delete(:workspace) || opts.delete(:wspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+    opts[:workspace_id] = wspace.id
+    search_term = opts.delete(:search_term)
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      if search_term && !search_term.empty?
+        column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Loot, search_term)
+        Mdm::Loot.includes(:host).where(opts).where(column_search_conditions)
+      else
+        Mdm::Loot.includes(:host).where(opts)
+      end
+    }
   end
+  alias_method :loot, :loots
 
   def report_loot(opts)
     return if not active
   ::ActiveRecord::Base.connection_pool.with_connection {
     wspace = opts.delete(:workspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
     path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required")
 
     host = nil
@@ -45,7 +61,7 @@ module Msf::DBManager::Loot
         host = opts[:host]
       else
         host = report_host({:workspace => wspace, :host => opts[:host]})
-        addr = normalize_host(opts[:host])
+        addr = Msf::Util::Host.normalize_host(opts[:host])
       end
     end
 
@@ -78,4 +94,45 @@ module Msf::DBManager::Loot
     ret[:loot] = loot
   }
   end
+
+  # Update the attributes of a Loot entry with the values in opts.
+  # The values in opts should match the attributes to update.
+  #
+  # @param opts [Hash] Hash containing the updated values. Key should match the attribute to update. Must contain :id of record to update.
+  # @return [Mdm::Loot] The updated Mdm::Loot object.
+  def update_loot(opts)
+    wspace = opts.delete(:workspace)
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+      opts[:workspace] = wspace
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      id = opts.delete(:id)
+      Mdm::Loot.update(id, opts)
+    }
+  end
+
+  # Deletes Loot entries based on the IDs passed in.
+  #
+  # @param opts[:ids] [Array] Array containing Integers corresponding to the IDs of the Loot entries to delete.
+  # @return [Array] Array containing the Mdm::Loot objects that were successfully deleted.
+  def delete_loot(opts)
+    raise ArgumentError.new("The following options are required: :ids") if opts[:ids].nil?
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      deleted = []
+      opts[:ids].each do |loot_id|
+        loot = Mdm::Loot.find(loot_id)
+        begin
+          deleted << loot.destroy
+        rescue # refs suck
+          elog("Forcibly deleting #{loot}")
+          deleted << loot.delete
+        end
+      end
+
+      return deleted
+    }
+  end
 end

lib/msf/core/db_manager/note.rb

@@ -68,7 +68,7 @@ module Msf::DBManager::Note
       if opts[:host].kind_of? ::Mdm::Host
         host = opts[:host]
       else
-        addr = normalize_host(opts[:host])
+        addr = Msf::Util::Host.normalize_host(opts[:host])
         host = report_host({:workspace => wspace, :host => addr})
       end
       # Do the same for a service if that's also included.

lib/msf/core/db_manager/service.rb

@@ -1,12 +1,21 @@
 module Msf::DBManager::Service
   # Deletes a port and associated vulns matching this port
-  def del_service(wspace, address, proto, port, comm='')
-
-    host = get_host(:workspace => wspace, :address => address)
-    return unless host
+  def delete_service(opts)
+    raise ArgumentError.new("The following options are required: :ids") if opts[:ids].nil?
 
   ::ActiveRecord::Base.connection_pool.with_connection {
-    host.services.where({:proto => proto, :port => port}).each { |s| s.destroy }
+    deleted = []
+    opts[:ids].each do |service_id|
+      service = Mdm::Service.find(service_id)
+      begin
+        deleted << service.destroy
+      rescue
+        elog("Forcibly deleting #{service.name}")
+        deleted << service.delete
+      end
+    end
+
+    return deleted
   }
   end
 
@@ -131,15 +140,28 @@ module Msf::DBManager::Service
   end
 
   # Returns a list of all services in the database
-  def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil)
+  def services(opts)
+    opts.delete(:workspace) # Mdm::Service apparently doesn't have an upstream Mdm::Workspace association
+    search_term = opts.delete(:search_term)
+    opts["hosts.address"] = opts.delete(:addresses)
+    opts.compact!
+
   ::ActiveRecord::Base.connection_pool.with_connection {
-    conditions = {}
-    conditions[:state] = [Msf::ServiceState::Open] if only_up
-    conditions[:proto] = proto if proto
-    conditions["hosts.address"] = addresses if addresses
-    conditions[:port] = ports if ports
-    conditions[:name] = names if names
-    wspace.services.includes(:host).where(conditions).order("hosts.address, port")
+    if search_term && !search_term.empty?
+      column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Service, search_term)
+      Mdm::Service.includes(:host).where(opts).where(column_search_conditions).order("hosts.address, port")
+    else
+      Mdm::Service.includes(:host).where(opts).order("hosts.address, port")
+    end
+  }
+  end
+
+  def update_service(opts)
+    opts.delete(:workspace) # Workspace isn't used with Mdm::Service. So strip it if it's present.
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    id = opts.delete(:id)
+    Mdm::Service.update(id, opts)
   }
   end
 end

lib/msf/core/db_manager/session.rb

@@ -1,4 +1,13 @@
 module Msf::DBManager::Session
+
+  def get_all_sessions()
+    return if not active
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      ::Mdm::Session.all
+    }
+  end
+
+
   # Returns a session based on opened_time, host address, and workspace
   # (or returns nil)
   def get_session(opts)
@@ -41,7 +50,7 @@ module Msf::DBManager::Session
   #     created, but the Mdm::Host will have been.   (There is no transaction
   #       to rollback the Mdm::Host creation.)
   #   @see #find_or_create_host
-  #   @see #normalize_host
+  #   @see #Msf::Util::Host.normalize_host
   #   @see #report_exploit_success
   #   @see #report_vuln
   #
@@ -101,7 +110,74 @@ module Msf::DBManager::Session
   }
   end
 
+  def report_session_host_dto(host_dto)
+    host_dto[:host] = get_host(host_dto)
+    create_mdm_session_from_host(host_dto)
+  end
+
+  def report_session_dto(session_dto)
+    return if not active
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      workspace = find_workspace(session_dto[:workspace])
+      host_data = session_dto[:host_data]
+      h_opts = {}
+      h_opts[:host]      = host_data[:host]
+      h_opts[:arch]      = host_data[:arch]
+      h_opts[:workspace] = workspace
+      host = find_or_create_host(h_opts)
+
+      session_data = session_dto[:session_data]
+      sess_data = {
+          datastore: session_data[:datastore],
+          desc: session_data[:desc],
+          host_id: host.id,
+          last_seen: session_dto[:time_stamp],
+          local_id: session_data[:local_id],
+          opened_at: session_dto[:time_stamp],
+          platform: session_data[:platform],
+          port: session_data[:port],
+          routes: [],
+          stype: session_data[:stype],
+          via_exploit: session_data[:via_exploit],
+          via_payload: session_data[:via_payload],
+      }
+
+      if sess_data[:via_exploit] == 'exploit/multi/handler' and sess_data[:datastore] and sess_data[:datastore]['ParentModule']
+        sess_data[:via_exploit] = sess_data[:datastore]['ParentModule']
+      end
+
+      session_db_record = ::Mdm::Session.create!(sess_data)
+
+      # TODO: Figure out task stuff
+
+
+      if sess_data[:via_exploit]
+        # This is a live session, we know the host is vulnerable to something.
+        infer_vuln_from_session_dto(session_dto, session_db_record, workspace)
+      end
+      session_db_record
+    }
+  end
+
+  # Clean out any stale sessions that have been orphaned by a dead framework instance.
+  # @param last_seen_interval [Integer] interval, in seconds, open sessions are marked as alive
+  def remove_stale_sessions(last_seen_interval)
+    return unless active
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      ::Mdm::Session.where(closed_at: nil).each do |db_session|
+        next unless db_session.last_seen.nil? or ((Time.now.utc - db_session.last_seen) > (2 * last_seen_interval))
+        db_session.closed_at    = db_session.last_seen || Time.now.utc
+        db_session.close_reason = "Orphaned"
+        db_session.save
+      end
+    }
+  end
+
+  #########
   protected
+  #########
 
   # @param session [Msf::Session] A session with a db_record Msf::Session#db_record
   # @param wspace [Mdm::Workspace]
@@ -117,10 +193,14 @@ module Msf::DBManager::Session
         mod_fullname = session.via_exploit
       end
       mod_detail = ::Mdm::Module::Detail.find_by_fullname(mod_fullname)
+
       if mod_detail.nil?
         # Then the cache isn't built yet, take the hit for instantiating the
         # module
         mod_detail = framework.modules.create(mod_fullname)
+        refs = mod_detail.references
+      else
+        refs = mod_detail.refs
       end
       mod_name = mod_detail.name
 
@@ -129,7 +209,7 @@ module Msf::DBManager::Session
         host: host,
         info: "Exploited by #{mod_fullname} to create Session #{s.id}",
         name: mod_name,
-        refs: mod_detail.refs.map(&:name),
+        refs: refs,
         workspace: wspace,
       }
 
@@ -143,7 +223,7 @@ module Msf::DBManager::Session
       attempt_info = {
         host: host,
         module: mod_fullname,
-        refs: mod_detail.refs,
+        refs: refs,
         service: service,
         session_id: s.id,
         timestamp: Time.now.utc,
@@ -166,7 +246,7 @@ module Msf::DBManager::Session
 
       wspace = opts[:workspace] || find_workspace(session.workspace)
       h_opts = { }
-      h_opts[:host]      = normalize_host(session)
+      h_opts[:host]      = Msf::Util::Host.normalize_host(session)
       h_opts[:arch]      = session.arch if session.respond_to?(:arch) and session.arch
       h_opts[:workspace] = wspace
       host = find_or_create_host(h_opts)
@@ -230,4 +310,46 @@ module Msf::DBManager::Session
     }
   end
 
+  def infer_vuln_from_session_dto(session_dto, session_db_record, workspace)
+    ::ActiveRecord::Base.connection_pool.with_connection {
+
+      vuln_info_dto = session_dto[:vuln_info]
+      host = session_db_record.host
+      mod_name = vuln_info_dto[:mod_name] || 'unknown'
+      refs = vuln_info_dto[:mod_references] || []
+
+      vuln_info = {
+          exploited_at: session_dto[:time_stamp],
+          host: host,
+          info: "Exploited by #{vuln_info_dto[:mod_fullname]} to create Session #{session_db_record.id}",
+          name: mod_name,
+          refs: refs,
+          workspace: workspace,
+      }
+
+      port    = vuln_info_dto[:remote_port]
+      service = (port ? host.services.find_by_port(port.to_i) : nil)
+
+      vuln_info[:service] = service if service
+
+      vuln = framework.db.report_vuln(vuln_info)
+
+      attempt_info = {
+          host: host,
+          module: vuln_info_dto[:mod_fullname],
+          refs: refs,
+          service: service,
+          session_id: session_db_record.id,
+          timestamp: session_dto[:time_stamp],
+          username: vuln_info_dto[:username],
+          vuln: vuln,
+          workspace: workspace,
+          run_id: vuln_info_dto[:run_id]
+      }
+
+      framework.db.report_exploit_success(attempt_info)
+
+      vuln
+    }
+  end
 end

lib/msf/core/db_manager/session_event.rb

@@ -1,4 +1,17 @@
 module Msf::DBManager::SessionEvent
+
+  def session_events(opts)
+    wspace = opts[:workspace] || opts[:wspace] || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      conditions = {}
+
+      Mdm::SessionEvent.all
+    }
+  end
   #
   # Record a session event in the database
   #

lib/msf/core/db_manager/vuln.rb

@@ -46,7 +46,7 @@ module Msf::DBManager::Vuln
 
   def find_vuln_by_refs(refs, host, service=nil)
     ref_ids = refs.find_all { |ref| ref.name.starts_with? 'CVE-'}
-    relation = host.vulns.includes(:refs)
+    relation = host.vulns.joins(:refs)
     if !service.try(:id).nil?
       return relation.where(service_id: service.try(:id), refs: { id: ref_ids}).first
     end
@@ -105,6 +105,8 @@ module Msf::DBManager::Vuln
       opts[:refs].each do |r|
         if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val))
           r = "#{r.ctx_id}-#{r.ctx_val}"
+        elsif (r.is_a?(Hash) and r[:ctx_id] and r[:ctx_val])
+          r = "#{r[:ctx_id]}-#{r[:ctx_val]}"
         end
         rids << find_or_create_ref(:name => r)
       end
@@ -116,7 +118,7 @@ module Msf::DBManager::Vuln
       host = opts[:host]
     else
       host = report_host({:workspace => wspace, :host => opts[:host]})
-      addr = normalize_host(opts[:host])
+      addr = Msf::Util::Host.normalize_host(opts[:host])
     end
 
     ret = {}
@@ -233,9 +235,63 @@ module Msf::DBManager::Vuln
   #
   # This methods returns a list of all vulnerabilities in the database
   #
-  def vulns(wspace=workspace)
-  ::ActiveRecord::Base.connection_pool.with_connection {
-    wspace.vulns
-  }
+  def vulns(opts)
+    wspace = opts.delete(:workspace) || opts.delete(:wspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+
+      search_term = opts.delete(:search_term)
+      if search_term && !search_term.empty?
+        column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Vuln, search_term)
+        wspace.vulns.includes(:host).where(opts).where(column_search_conditions)
+      else
+        wspace.vulns.includes(:host).where(opts)
+      end
+    }
+  end
+
+  # Update the attributes of a Vuln entry with the values in opts.
+  # The values in opts should match the attributes to update.
+  #
+  # @param opts [Hash] Hash containing the updated values. Key should match the attribute to update. Must contain :id of record to update.
+  # @return [Mdm::Vuln] The updated Mdm::Vuln object.
+  def update_vuln(opts)
+    # process workspace string for update if included in opts
+    wspace = opts.delete(:workspace)
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+      opts[:workspace] = wspace
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      id = opts.delete(:id)
+      Mdm::Vuln.update(id, opts)
+    }
+  end
+
+  # Deletes Vuln entries based on the IDs passed in.
+  #
+  # @param opts[:ids] [Array] Array containing Integers corresponding to the IDs of the Vuln entries to delete.
+  # @return [Array] Array containing the Mdm::Vuln objects that were successfully deleted.
+  def delete_vuln(opts)
+    raise ArgumentError.new("The following options are required: :ids") if opts[:ids].nil?
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      deleted = []
+      opts[:ids].each do |vuln_id|
+        vuln = Mdm::Vuln.find(vuln_id)
+        begin
+          deleted << vuln.destroy
+        rescue # refs suck
+          elog("Forcibly deleting #{vuln}")
+          deleted << vuln.delete
+        end
+      end
+
+      return deleted
+    }
   end
 end

lib/msf/core/db_manager/vuln_attempt.rb

@@ -17,4 +17,25 @@ module Msf::DBManager::VulnAttempt
     vuln.vuln_attempts.create(info)
   }
   end
+
+  #
+  # This methods returns a list of all vulnerability attempts in the database
+  #
+  def vuln_attempts(opts)
+    wspace = opts.delete(:workspace) || opts.delete(:wspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+
+      search_term = opts.delete(:search_term)
+      if search_term && !search_term.empty?
+        column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::VulnAttempt, search_term)
+        Mdm::VulnAttempt.where(opts).where(column_search_conditions)
+      else
+        Mdm::VulnAttempt.where(opts)
+      end
+    }
+  end
 end

lib/msf/core/db_manager/workspace.rb

@@ -33,4 +33,90 @@ module Msf::DBManager::Workspace
     ::Mdm::Workspace.order('updated_at asc').load
   }
   end
+
+  #
+  # Returns an array of all the associated workspace records counts.
+  #
+  def workspace_associations_counts()
+    results = Array.new()
+
+    ::ActiveRecord::Base.connection_pool.with_connection {
+      workspaces.each do |ws|
+        results << {
+            :name  => ws.name,
+            :hosts_count => ws.hosts.count,
+            :services_count => ws.services.count,
+            :vulns_count => ws.vulns.count,
+            :creds_count => ws.core_credentials.count,
+            :loots_count => ws.loots.count,
+            :notes_count => ws.notes.count
+        }
+      end
+    }
+
+    return results
+  end
+
+  def delete_all_workspaces()
+    return delete_workspaces(workspaces.map(&:name))
+  end
+
+  def delete_workspaces(names)
+    status_msg = []
+    error_msg = []
+
+    switched = false
+    # Delete workspaces
+    names.each do |name|
+      workspace = framework.db.find_workspace(name)
+      if workspace.nil?
+        error << "Workspace not found: #{name}"
+      elsif workspace.default?
+        workspace.destroy
+        workspace = framework.db.add_workspace(name)
+        status_msg << 'Deleted and recreated the default workspace'
+      else
+        # switch to the default workspace if we're about to delete the current one
+        if framework.db.workspace.name == workspace.name
+          framework.db.workspace = framework.db.default_workspace
+          switched = true
+        end
+        # now destroy the named workspace
+        workspace.destroy
+        status_msg << "Deleted workspace: #{name}"
+      end
+    end
+    (status_msg << "Switched workspace: #{framework.db.workspace.name}") if switched
+    return status_msg, error_msg
+  end
+
+  #
+  # Renames a workspace
+  #
+  def rename_workspace(from_name, to_name)
+    raise "Workspace exists: #{to_name}" if framework.db.find_workspace(to_name)
+
+    workspace = find_workspace(from_name)
+    raise "Workspace not found: #{name}" if workspace.nil?
+
+    workspace.name = new
+    workspace.save!
+
+    # Recreate the default workspace to avoid errors
+    if workspace.default?
+      framework.db.add_workspace(from_name)
+      #print_status("Recreated default workspace after rename")
+    end
+
+    # Switch to new workspace if old name was active
+    if (@workspace_name == workspace.name)
+      framework.db.workspace = workspace
+      #print_status("Switched workspace: #{framework.db.workspace.name}")
+    end
+  end
+
+  def get_workspace(opts)
+    workspace = opts.delete(:wspace) || opts.delete(:workspace) || workspace
+    find_workspace(workspace) if (workspace.is_a?(String))
+  end
 end

lib/msf/core/framework.rb

@@ -58,7 +58,7 @@ class Framework
   require 'msf/core/module_manager'
   require 'msf/core/session_manager'
   require 'msf/core/plugin_manager'
-  require 'msf/core/db_manager'
+  require 'metasploit/framework/data_service/proxy/core'
   require 'msf/core/event_dispatcher'
   require 'rex/json_hash_file'
   require 'msf/core/cert_provider'
@@ -193,13 +193,13 @@ class Framework
   #
   attr_reader   :browser_profiles
 
-  # The framework instance's db manager. The db manager
-  # maintains the database db and handles db events
   #
-  # @return [Msf::DBManager]
+  # The framework instance's data service proxy
+  #
+  # @return [Metasploit::Framework::DataService::DataProxy]
   def db
     synchronize {
-      @db ||= Msf::DBManager.new(self, options)
+      @db ||= get_db
     }
   end
 
@@ -268,6 +268,19 @@ protected
   attr_writer   :db # :nodoc:
   attr_writer   :uuid_db # :nodoc:
   attr_writer   :browser_profiles # :nodoc:
+
+  private
+
+  def get_db
+    if !options['DisableDatabase']
+      db_manager = Msf::DBManager.new(self)
+      db_manager.init_db(options)
+      options[:db_manager] = db_manager
+    end
+
+    Metasploit::Framework::DataService::DataProxy.new(options)
+  end
+
 end
 
 class FrameworkEventSubscriber
@@ -326,7 +339,7 @@ class FrameworkEventSubscriber
   ##
   # :category: ::Msf::UiEventSubscriber implementors
   def on_ui_command(command)
-    if framework.db.active
+    if (framework.db and framework.db.active)
       report_event(:name => "ui_command", :info => {:command => command})
     end
   end
@@ -334,7 +347,7 @@ class FrameworkEventSubscriber
   ##
   # :category: ::Msf::UiEventSubscriber implementors
   def on_ui_stop()
-    if framework.db.active
+    if (framework.db and framework.db.active)
       report_event(:name => "ui_stop")
     end
   end

lib/msf/core/session_manager.rb

@@ -20,7 +20,7 @@ class SessionManager < Hash
 
   include Framework::Offspring
 
-  LAST_SEEN_INTERVAL  = 60 * 2.5
+  LAST_SEEN_INTERVAL = 60 * 2.5
   SCHEDULER_THREAD_COUNT = 5
 
   def initialize(framework)
@@ -98,6 +98,11 @@ class SessionManager < Hash
           end
         end
 
+        #
+        # Skip the database cleanup code below if there is no database
+        #
+        next unless framework.db && framework.db.active && framework.db.is_local?
+
         #
         # Mark all open session as alive every LAST_SEEN_INTERVAL
         #
@@ -107,40 +112,26 @@ class SessionManager < Hash
           # processing time for large session lists from skewing our update interval.
 
           last_seen_timer = Time.now.utc
-          if framework.db.active
-            ::ActiveRecord::Base.connection_pool.with_connection do
-              values.each do |s|
-                # Update the database entry on a regular basis, marking alive threads
-                # as recently seen.  This notifies other framework instances that this
-                # session is being maintained.
-                if s.db_record
-                  s.db_record.last_seen = Time.now.utc
-                  s.db_record.save
-                end
+
+          ::ActiveRecord::Base.connection_pool.with_connection do
+            values.each do |s|
+              # Update the database entry on a regular basis, marking alive threads
+              # as recently seen.  This notifies other framework instances that this
+              # session is being maintained.
+              if s.db_record
+                s.db_record.last_seen = Time.now.utc
+                s.db_record.save
               end
             end
           end
         end
 
-
-        #
-        # Skip the database cleanup code below if there is no database
-        #
-        next if not (framework.db and framework.db.active)
-
         #
         # Clean out any stale sessions that have been orphaned by a dead
         # framework instance.
         #
-        ::ActiveRecord::Base.connection_pool.with_connection do |conn|
-          ::Mdm::Session.where(closed_at: nil).each do |db_session|
-            if db_session.last_seen.nil? or ((Time.now.utc - db_session.last_seen) > (2*LAST_SEEN_INTERVAL))
-              db_session.closed_at    = db_session.last_seen || Time.now.utc
-              db_session.close_reason = "Orphaned"
-              db_session.save
-            end
-          end
-        end
+        framework.db.remove_stale_sessions(LAST_SEEN_INTERVAL)
+
       end
 
       #

lib/msf/core/thread_manager.rb

@@ -108,7 +108,7 @@ class ThreadManager < Array
           elog("Call Stack\n#{e.backtrace.join("\n")}")
           raise e
         ensure
-          if framework.db and framework.db.active
+          if framework.db && framework.db.active && framework.db.is_local?
             # NOTE: despite the Deprecation Warning's advice, this should *NOT*
             # be ActiveRecord::Base.connection.close which causes unrelated
             # threads to raise ActiveRecord::StatementInvalid exceptions at

lib/msf/ui/console/command_dispatcher/creds.rb

@@ -290,6 +290,7 @@ class Creds
     port_ranges   = []
     svcs          = []
     rhosts        = []
+    opts          = {}
 
     set_rhosts = false
 
@@ -314,6 +315,7 @@ class Creds
         end
       when "-t","--type"
         ptype = args.shift
+        opts[:ptype] = ptype
         if (!ptype)
           print_error("Argument required for -t")
           return
@@ -325,14 +327,17 @@ class Creds
           return
         end
         svcs = service.split(/[\s]*,[\s]*/)
+        opts[:svcs] = svcs
       when "-P","--password"
         pass = args.shift
+        opts[:pass] = pass
         if (!pass)
           print_error("Argument required for -P")
           return
         end
       when "-u","--user"
         user = args.shift
+        opts[:user] = user
         if (!user)
           print_error("Argument required for -u")
           return
@@ -343,6 +348,7 @@ class Creds
         set_rhosts = true
       when '-O', '--origins'
         hosts = args.shift
+        opts[:hosts] = hosts
         if !hosts
           print_error("Argument required for -O")
           return
@@ -350,6 +356,7 @@ class Creds
         arg_host_range(hosts, origin_ranges)
       when '-S', '--search-term'
         search_term = args.shift
+        opts[:search_term] = search_term
       else
         # Anything that wasn't an option is a host to search for
         unless (arg_host_range(arg, host_ranges))
@@ -374,6 +381,8 @@ class Creds
              end
     end
 
+    opts[:type] = type if type
+
     # normalize
     ports = port_ranges.flatten.uniq
     svcs.flatten!
@@ -384,120 +393,98 @@ class Creds
     }
 
     tbl = Rex::Text::Table.new(tbl_opts)
+    opts[:wspace] = framework.db.workspace
+    query = framework.db.creds(opts)
 
-    ::ActiveRecord::Base.connection_pool.with_connection {
-      query = Metasploit::Credential::Core.where( workspace_id: framework.db.workspace )
-      query = query.includes(:private, :public, :logins).references(:private, :public, :logins)
-      query = query.includes(logins: [ :service, { service: :host } ])
+    query.each do |core|
 
-      if type.present?
-        query = query.where(metasploit_credential_privates: { type: type })
+      # Exclude non-blank username creds if that's what we're after
+      if user == "" && core.public && !(core.public.username.blank?)
+        next
       end
 
-      if svcs.present?
-        query = query.where(Mdm::Service[:name].in(svcs))
+      # Exclude non-blank password creds if that's what we're after
+      if pass == "" && core.private && !(core.private.data.blank?)
+        next
       end
 
-      if ports.present?
-        query = query.where(Mdm::Service[:port].in(ports))
+      origin = ''
+      if core.origin.kind_of?(Metasploit::Credential::Origin::Service)
+        origin = core.origin.service.host.address
+      elsif core.origin.kind_of?(Metasploit::Credential::Origin::Session)
+        origin = core.origin.session.host.address
       end
 
-      if user.present?
-        # If we have a user regex, only include those that match
-        query = query.where('"metasploit_credential_publics"."username" ~* ?', user)
+      if !origin.empty? && origin_ranges.present? && !origin_ranges.any? {|range| range.include?(origin) }
+        next
       end
 
-      if pass.present?
-        # If we have a password regex, only include those that match
-        query = query.where('"metasploit_credential_privates"."data" ~* ?', pass)
-      end
+      if core.logins.empty? && origin_ranges.empty?
+        public_val = core.public ? core.public.username : ""
+        private_val = core.private ? core.private.data : ""
+        realm_val = core.realm ? core.realm.value : ""
+        human_val = core.private ? core.private.class.model_name.human : ""
 
-      if host_ranges.any? || ports.any? || svcs.any?
-        # Only find Cores that have non-zero Logins if the user specified a
-        # filter based on host, port, or service name
-        query = query.where(Metasploit::Credential::Login[:id].not_eq(nil))
-      end
-
-      query.find_each do |core|
-
-        # Exclude non-blank username creds if that's what we're after
-        if user == "" && core.public && !(core.public.username.blank?)
-          next
-        end
-
-        # Exclude non-blank password creds if that's what we're after
-        if pass == "" && core.private && !(core.private.data.blank?)
-          next
-        end
-
-        origin = ''
-        if core.origin.kind_of?(Metasploit::Credential::Origin::Service)
-          origin = core.origin.service.host.address
-        elsif core.origin.kind_of?(Metasploit::Credential::Origin::Session)
-          origin = core.origin.session.host.address
-        end
-
-        if !origin.empty? && origin_ranges.present? && !origin_ranges.any? {|range| range.include?(origin) }
-          next
-        end
-
-        if core.logins.empty? && origin_ranges.empty?
-          tbl << [
-            "", # host
-            "", # cred
-            "", # service
-            core.public,
-            core.private,
-            core.realm,
-            core.private ? core.private.class.model_name.human : "",
-          ]
-        else
-          core.logins.each do |login|
-            # If none of this Core's associated Logins is for a host within
-            # the user-supplied RangeWalker, then we don't have any reason to
-            # print it out. However, we treat the absence of ranges as meaning
-            # all hosts.
-            if host_ranges.present? && !host_ranges.any? { |range| range.include?(login.service.host.address) }
-              next
-            end
-
-            row = [ login.service.host.address ]
-            row << origin
-            rhosts << login.service.host.address
-            if login.service.name.present?
-              row << "#{login.service.port}/#{login.service.proto} (#{login.service.name})"
-            else
-              row << "#{login.service.port}/#{login.service.proto}"
-            end
-
-            row += [
-              core.public,
-              core.private,
-              core.realm,
-              core.private ? core.private.class.model_name.human : "",
-            ]
-            tbl << row
-          end
-        end
-        if mode == :delete
-          core.destroy
-          delete_count += 1
-        end
-      end
-
-      if output_file.nil?
-        print_line(tbl.to_s)
+        tbl << [
+          "", # host
+          "", # cred
+          "", # service
+          public_val,
+          private_val,
+          realm_val,
+          human_val
+        ]
       else
-        # create the output file
-        ::File.open(output_file, "wb") { |f| f.write(tbl.to_csv) }
-        print_status("Wrote creds to #{output_file}")
-      end
+        core.logins.each do |login|
+          # If none of this Core's associated Logins is for a host within
+          # the user-supplied RangeWalker, then we don't have any reason to
+          # print it out. However, we treat the absence of ranges as meaning
+          # all hosts.
+          if host_ranges.present? && !host_ranges.any? { |range| range.include?(login.service.host.address) }
+            next
+          end
 
-      # Finally, handle the case where the user wants the resulting list
-      # of hosts to go into RHOSTS.
-      set_rhosts_from_addrs(rhosts.uniq) if set_rhosts
-      print_status("Deleted #{delete_count} creds") if delete_count > 0
-    }
+          row = [ login.service.host.address ]
+          row << origin
+          rhosts << login.service.host.address
+          if login.service.name.present?
+            row << "#{login.service.port}/#{login.service.proto} (#{login.service.name})"
+          else
+            row << "#{login.service.port}/#{login.service.proto}"
+          end
+
+          public_val = core.public ? core.public.username : ""
+          private_val = core.private ? core.private.data : ""
+          realm_val = core.realm ? core.realm.value : ""
+          human_val = core.private ? core.private.class.model_name.human : ""
+
+          row += [
+            public_val,
+            private_val,
+            realm_val,
+            human_val
+          ]
+          tbl << row
+        end
+      end
+      if mode == :delete
+        core.destroy
+        delete_count += 1
+      end
+    end
+
+    if output_file.nil?
+      print_line(tbl.to_s)
+    else
+      # create the output file
+      ::File.open(output_file, "wb") { |f| f.write(tbl.to_csv) }
+      print_status("Wrote creds to #{output_file}")
+    end
+
+    # Finally, handle the case where the user wants the resulting list
+    # of hosts to go into RHOSTS.
+    set_rhosts_from_addrs(rhosts.uniq) if set_rhosts
+    print_status("Deleted #{delete_count} creds") if delete_count > 0
   end
 
   def cmd_creds_tabs(str, words)

lib/msf/ui/console/driver.rb

@@ -122,12 +122,11 @@ class Driver < Msf::Ui::Driver
       enstack_dispatcher(dispatcher)
     end
 
-    # Add the database dispatcher if it is usable
-    if (framework.db.usable)
-      require 'msf/ui/console/command_dispatcher/db'
-      enstack_dispatcher(CommandDispatcher::Db)
-      require 'msf/ui/console/command_dispatcher/creds'
-      enstack_dispatcher(CommandDispatcher::Creds)
+    if framework.db && framework.db.active
+        require 'msf/ui/console/command_dispatcher/db'
+        enstack_dispatcher(CommandDispatcher::Db)
+        require 'msf/ui/console/command_dispatcher/creds'
+        enstack_dispatcher(CommandDispatcher::Creds)
     else
       print_error("***")
       if framework.db.error == "disabled"
@@ -196,7 +195,7 @@ class Driver < Msf::Ui::Driver
       self.framework.init_module_paths(module_paths: opts['ModulePath'])
     end
 
-    if !opts['DeferModuleLoads']
+    if framework.db && framework.db.active && framework.db.is_local? && !opts['DeferModuleLoads']
       framework.threads.spawn("ModuleCacheRebuild", true) do
         framework.modules.refresh_cache_from_module_files
       end

lib/msf/util.rb

@@ -22,3 +22,9 @@ end
 # Executable generation and encoding
 require 'msf/util/exe'
 require 'msf/util/helper'
+
+# Host helpers
+require 'msf/util/host'
+
+# DBManager helpers
+require 'msf/util/db_manager'

lib/msf/util/db_manager.rb

@@ -0,0 +1,19 @@
+module Msf
+  module Util
+    module DBManager
+      # Creates search conditions to match the specified search string against all of the model's columns.
+      #
+      # @param model - An ActiveRecord model object
+      # @param search - A string regex search
+      # @return Arel::Nodes::Or object that represents a search of all of the model's columns
+      def self.create_all_column_search_conditions(model, search)
+        search = "(?mi)#{search}"
+        condition_set = model.columns.map do |column|
+          Arel::Nodes::Regexp.new(Arel::Nodes::NamedFunction.new("CAST", [model.arel_table[column.name].as("TEXT")]),
+                                  Arel::Nodes.build_quoted(search))
+        end
+        condition_set.reduce { |conditions, condition| conditions.or(condition).expr }
+      end
+    end
+  end
+end

lib/msf/util/host.rb

@@ -0,0 +1,62 @@
+# -*- coding: binary -*-
+
+module Msf
+  module Util
+    module Host
+
+      #
+      # Returns something suitable for the +:host+ parameter to the various report_* methods
+      #
+      # Takes a Host object, a Session object, an Msf::Session object or a String
+      # address
+      #
+      def self.normalize_host(host)
+        return host if defined?(::Mdm) && host.kind_of?(::Mdm::Host)
+        norm_host = nil
+
+        if (host.kind_of? String)
+
+          if Rex::Socket.is_ipv4?(host)
+            # If it's an IPv4 addr with a port on the end, strip the port
+            if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/
+              norm_host = $1
+            else
+              norm_host = host
+            end
+          elsif Rex::Socket.is_ipv6?(host)
+            # If it's an IPv6 addr, drop the scope
+            address, scope = host.split('%', 2)
+            norm_host = address
+          else
+            norm_host = Rex::Socket.getaddress(host, true)
+          end
+        elsif defined?(::Mdm) && host.kind_of?(::Mdm::Session)
+          norm_host = host.host
+        elsif host.respond_to?(:session_host)
+          # Then it's an Msf::Session object
+          norm_host = host.session_host
+        end
+
+        # If we got here and don't have a norm_host yet, it could be a
+        # Msf::Session object with an empty or nil tunnel_host and tunnel_peer;
+        # see if it has a socket and use its peerhost if so.
+        if (
+        norm_host.nil? &&
+            host.respond_to?(:sock) &&
+            host.sock.respond_to?(:peerhost) &&
+            host.sock.peerhost.to_s.length > 0
+        )
+          norm_host = session.sock.peerhost
+        end
+        # If We got here and still don't have a real host, there's nothing left
+        # to try, just log it and return what we were given
+        if !norm_host
+          dlog("Host could not be normalized: #{host.inspect}")
+          norm_host = host
+        end
+
+        norm_host
+      end
+    end
+  end
+end

metasploit-framework.gemspec

@@ -100,6 +100,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'redcarpet'
   # Needed for Microsoft patch finding tool (msu_finder)
   spec.add_runtime_dependency 'patch_finder'
+  # Required for msfdb_ws (Metasploit data base as a webservice)
+  spec.add_runtime_dependency 'thin'
+  spec.add_runtime_dependency 'sinatra'
   # TimeZone info
   spec.add_runtime_dependency 'tzinfo-data'
   # Gem for dealing with SSHKeys