Pass 1: externalize database

Gemfile

@@ -3,6 +3,9 @@ source 'https://rubygems.org'
 #   spec.add_runtime_dependency '<name>', [<version requirements>]
 gemspec name: 'metasploit-framework'
 
+gem 'thin'
+gem 'sinatra'
+gem 'ruby-prof'
 gem 'bit-struct', git: 'https://github.com/busterb/bit-struct', branch: 'ruby-2.4'
 gem 'method_source', git: 'https://github.com/banister/method_source', branch: 'master'
 
@@ -24,6 +27,8 @@ group :development do
   # metasploit-aggregator as a framework only option for now
   # Metasploit::Aggregator external session proxy
   gem 'metasploit-aggregator'
+
+  #gem 'rex-core', path: '/home/chlee/rapid7/rex-core'
 end
 
 group :development, :test do

lib/metasploit/framework/command/base.rb

@@ -44,7 +44,11 @@ class Metasploit::Framework::Command::Base
   #
   # @return (see parsed_options)
   def self.require_environment!
+    # TODO: Look into removing Rails.application (save ~20mb)
+    # return self.parsed_options if ( self.parsed_options.options.database.remote_process)
+
     parsed_options = self.parsed_options
+
     # RAILS_ENV must be set before requiring 'config/application.rb'
     parsed_options.environment!
     ARGV.replace(parsed_options.positional)
@@ -79,7 +83,9 @@ class Metasploit::Framework::Command::Base
 
   def self.start
     parsed_options = require_environment!
-    new(application: Rails.application, parsed_options: parsed_options).start
+    is_db_remote = false # parsed_options.options.database.remote_process
+    application = is_db_remote ? nil : Rails.application
+    new(application: application, parsed_options: parsed_options).start
   end
 
   #

lib/metasploit/framework/command/console.rb

@@ -79,6 +79,7 @@ class Metasploit::Framework::Command::Console < Metasploit::Framework::Command::
       driver_options['DatabaseEnv'] = options.environment
       driver_options['DatabaseMigrationPaths'] = options.database.migrations_paths
       driver_options['DatabaseYAML'] = options.database.config
+      driver_options['DatabaseRemoteProcess'] = options.database.remote_process
       driver_options['DeferModuleLoads'] = options.modules.defer_loads
       driver_options['DisableBanner'] = options.console.quiet
       driver_options['DisableDatabase'] = options.database.disable

lib/metasploit/framework/data_service.rb

@@ -0,0 +1,32 @@
+require 'metasploit/framework/data_service/stubs/host_data_service'
+require 'metasploit/framework/data_service/stubs/vuln_data_service'
+require 'metasploit/framework/data_service/stubs/event_data_service'
+require 'metasploit/framework/data_service/stubs/workspace_data_service'
+require 'metasploit/framework/data_service/stubs/note_data_service'
+require 'metasploit/framework/data_service/stubs/web_data_service'
+require 'metasploit/framework/data_service/stubs/service_data_service'
+
+#
+# All data service implementations should include this module to ensure proper implementation
+#
+module Metasploit
+module Framework
+module DataService
+  include HostDataService
+  include EventDataService
+  include VulnDataService
+  include WorkspaceDataService
+  include WebDataService
+  include NoteDataService
+  include ServiceDataService
+
+  def name
+    raise 'DataLService#name is not implemented';
+  end
+
+  def active
+    raise 'DataLService#active is not implemented';
+  end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/core.rb

@@ -0,0 +1,202 @@
+require 'singleton'
+require 'open3'
+require 'rex/ui'
+require 'rex/logging'
+require 'msf/core/db_manager'
+require 'metasploit/framework/data_service/remote/http/core'
+require 'metasploit/framework/data_service/remote/http/remote_service_endpoint'
+require 'metasploit/framework/data_service/proxy/data_proxy_auto_loader'
+
+#
+# Holds references to data services (@see Metasploit::Framework::DataService)
+# and forwards data to the implementation set as current.
+#
+module Metasploit
+module Framework
+module DataService
+class DataProxy
+  include Singleton
+  include DataProxyAutoLoader
+
+  attr_reader :usable
+
+  #
+  # Returns current error state
+  #
+  def error
+    return @error if (@error)
+    return @data_service.error if @data_service
+    return "none"
+  end
+
+  #
+  # Determines if the data service is active
+  #
+  def active
+    if (@data_service)
+      return @data_service.active
+    end
+
+    return false
+  end
+
+  #
+  # Initializes the data service to be used - primarily on startup
+  #
+  def init(framework, opts)
+    @mutex.synchronize {
+      if (@initialized)
+        return
+      end
+
+      begin
+        if (opts['DisableDatabase'])
+          @error = 'disabled'
+          return
+        elsif (opts['DatabaseRemoteProcess'])
+          run_remote_db_process(opts)
+        else
+          run_local_db_process(framework, opts)
+        end
+        @usable = true
+        @initialized = true
+      rescue Exception => e
+        puts "Unable to initialize a dataservice #{e.message}"
+        return
+      end
+    }
+
+  end
+
+  #
+  # Registers a data service with the proxy and immediately
+  # set as primary if online
+  #
+  def register_data_service(data_service, online=false)
+    validate(data_service)
+
+    puts "Registering data service: #{data_service.name}"
+    data_service_id = @data_service_id += 1
+    @data_services[data_service_id] = data_service
+    set_data_service(data_service_id, online)
+  end
+
+  #
+  # Set the data service to be used
+  #
+  def set_data_service(data_service_id, online=false)
+    data_service = @data_services[data_service_id.to_i]
+    if (data_service.nil?)
+      puts "Data service with id: #{data_service_id} does not exist"
+      return nil
+    end
+
+    if (!online && !data_service.active)
+      puts "Data service not online: #{data_service.name}, not setting as active"
+      return nil
+    end
+
+    puts "Setting active data service: #{data_service.name}"
+    @data_service = data_service
+  end
+
+  #
+  # Prints out a list of the current data services
+  #
+  def print_data_services()
+    @data_services.each_key {|key|
+      out = "id: #{key}, description: #{@data_services[key].name}"
+      if (!@data_service.nil? && @data_services[key].name == @data_service.name)
+        out += " [active]"
+      end
+      puts out  #hahaha
+    }
+  end
+
+  #
+  # Used to bridge the local db
+  #
+  def method_missing(method, *args, &block)
+    puts "Attempting to delegate method: #{method}"
+    unless @data_service.nil?
+      @data_service.send(method, *args, &block)
+    end
+  end
+
+  #
+  # Attempt to shutdown the local db process if it exists
+  #
+  def exit_called
+    if @pid
+      puts 'Killing db process'
+      begin
+        Process.kill("TERM", @pid)
+      rescue Exception => e
+        puts "Unable to kill db process: #{e.message}"
+      end
+    end
+  end
+
+  #########
+  protected
+  #########
+
+  def get_data_service
+    raise 'No registered data_service' unless @data_service
+    return @data_service
+  end
+
+  #######
+  private
+  #######
+
+  def initialize
+    @data_services = {}
+    @data_service_id = 0
+    @usable = false
+    @initialized = false
+    @mutex = Mutex.new()
+  end
+
+  def validate(data_service)
+    raise "Invalid data_service: #{data_service.class}, not of type Metasploit::Framework::DataService" unless data_service.is_a? (Metasploit::Framework::DataService)
+    raise 'Cannot register null data service data_service' unless data_service
+    raise 'Data Service already exists' if data_service_exist?(data_service)
+  end
+
+  def data_service_exist?(data_service)
+    @data_services.each_value{|value|
+      if (value.name == data_service.name)
+        return true
+      end
+    }
+
+    return false
+  end
+
+
+  def run_local_db_process(framework, opts)
+    puts 'Initializing local db process'
+    db_manager = Msf::DBManager.new(framework)
+    if (db_manager.usable and not opts['SkipDatabaseInit'])
+        register_data_service(db_manager, true)
+        db_manager.init_db(opts)
+    end
+  end
+
+  def run_remote_db_process(opts)
+    # started with no signal to prevent ctrl-c from taking out db
+    db_script = File.join( Msf::Config.install_root, "msfdb -ns")
+    wait_t = Open3.pipeline_start(db_script)
+    @pid = wait_t[0].pid
+    puts "Started process with pid #{@pid}"
+
+    endpoint = Metasploit::Framework::DataService::RemoteServiceEndpoint.new('localhost', 8080)
+    remote_host_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint)
+    register_data_service(remote_host_data_service, true)
+  end
+
+end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/data_proxy_auto_loader.rb

@@ -0,0 +1,20 @@
+#
+# Autoloads specific data proxies
+#
+module DataProxyAutoLoader
+  autoload :HostDataProxy, 'metasploit/framework/data_service/proxy/host_data_proxy'
+  autoload :VulnDataProxy, 'metasploit/framework/data_service/proxy/vuln_data_proxy'
+  autoload :EventDataProxy, 'metasploit/framework/data_service/proxy/event_data_proxy'
+  autoload :WorkspaceDataProxy, 'metasploit/framework/data_service/proxy/workspace_data_proxy'
+  autoload :NoteDataProxy, 'metasploit/framework/data_service/proxy/note_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :ServiceDataProxy, 'metasploit/framework/data_service/proxy/service_data_proxy'
+  include ServiceDataProxy
+  include HostDataProxy
+  include VulnDataProxy
+  include EventDataProxy
+  include WorkspaceDataProxy
+  include NoteDataProxy
+  include WebDataProxy
+end

lib/metasploit/framework/data_service/proxy/event_data_proxy.rb

@@ -0,0 +1,12 @@
+module EventDataProxy
+
+  def report_event(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_event(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_event threw exception: #{e.message}"
+    end
+  end
+
+end

lib/metasploit/framework/data_service/proxy/host_data_proxy.rb

@@ -0,0 +1,59 @@
+module HostDataProxy
+
+  def hosts(wspace, non_dead = false, addresses = nil)
+    begin
+      data_service = self.get_data_service()
+      opts = {}
+      opts[:wspace] = wspace
+      opts[:non_dead] = non_dead
+      opts[:addresses] = addresses
+      data_service.hosts(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#hosts threw exception: #{e.message}"
+    end
+  end
+
+  def find_or_create_host(opts)
+    puts 'Calling find host'
+    report_host(opts)
+  end
+
+  def report_host(opts)
+    return unless valid(opts)
+
+    begin
+      data_service = self.get_data_service()
+      data_service.report_host(opts)
+    rescue  Exception => e
+        puts"Call to  #{data_service.class}#report_host threw exception: #{e.message}"
+        opts.each do |key, value| puts "#{key} : #{value}" end
+    end
+  end
+
+  def report_hosts(hosts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_hosts(hosts)
+    rescue  Exception => e
+      puts "Call to  #{data_service.class}#report_hosts threw exception: #{e.message}"
+    end
+  end
+
+  private
+
+  def valid(opts)
+    unless opts[:host]
+      puts 'Invalid host hash passed, :host is missing'
+      return false
+    end
+
+    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
+    if opts[:host].eql? "Remote Pipe"
+      puts "Invalid host hash passed, address was of type 'Remote Pipe'"
+      return false
+    end
+
+    return true
+  end
+
+end

lib/metasploit/framework/data_service/proxy/note_data_proxy.rb

@@ -0,0 +1,10 @@
+module NoteDataProxy
+  def report_note(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_note(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_note threw exception: #{e.message}"
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/service_data_proxy.rb

@@ -0,0 +1,12 @@
+module ServiceDataProxy
+
+  def report_service(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_service(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_service threw exception: #{e.message}"
+    end
+  end
+
+end

lib/metasploit/framework/data_service/proxy/vuln_data_proxy.rb

@@ -0,0 +1,12 @@
+module VulnDataProxy
+
+  def report_vuln(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_vuln(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_vuln threw exception: #{e.message}"
+    end
+  end
+
+end

lib/metasploit/framework/data_service/proxy/web_data_proxy.rb

@@ -0,0 +1,10 @@
+module WebDataProxy
+  def report_web_site(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_web_site(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_web_site threw exception: #{e.message}"
+    end
+  end
+end

lib/metasploit/framework/data_service/proxy/workspace_data_proxy.rb

@@ -0,0 +1,56 @@
+module WorkspaceDataProxy
+
+  def find_workspace(workspace_name)
+    begin
+      data_service = self.get_data_service()
+      data_service.find_workspace(workspace_name)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#find_workspace threw exception: #{e.message}"
+    end
+  end
+
+  def add_workspace(workspace_name)
+    begin
+      data_service = self.get_data_service()
+      data_service.add_workspace(workspace_name)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#add_workspace threw exception: #{e.message}"
+    end
+  end
+
+  def default_workspace
+    begin
+      data_service = self.get_data_service()
+      data_service.default_workspace
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#default_workspace threw exception: #{e.message}"
+    end
+  end
+
+  def workspace
+    begin
+      data_service = self.get_data_service()
+      data_service.workspace
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#workspace threw exception: #{e.message}"
+    end
+  end
+
+  def workspace=(workspace)
+    begin
+      data_service = self.get_data_service()
+      data_service.workspace = workspace
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#find_workspace threw exception: #{e.message}"
+    end
+  end
+
+  def workspaces
+    begin
+      data_service = self.get_data_service()
+      data_service.workspaces
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#workspaces threw exception: #{e.message}"
+    end
+  end
+end

lib/metasploit/framework/data_service/remote/http/core.rb

@@ -0,0 +1,237 @@
+require 'metasploit/framework/data_service/remote/http/remote_service_endpoint'
+require 'metasploit/framework/data_service'
+require 'metasploit/framework/data_service/remote/http/data_service_auto_loader'
+
+#
+# Parent data service for managing metasploit data in/on a separate process/machine over HTTP(s)
+#
+module Metasploit
+module Framework
+module DataService
+class RemoteHTTPDataService
+  include Metasploit::Framework::DataService
+  include DataServiceAutoLoader
+
+  ONLINE_TEST_URL = "/api/1/msf/online"
+  EXEC_ASYNC = {:exec_async => true}
+  GET_REQUEST = 'GET'
+  POST_REQUEST = 'POST'
+
+  #
+  # @param endpoint - A RemoteServiceEndpoint. Cannot be nil
+  #
+  def initialize(endpoint)
+    validate_endpoint(endpoint)
+    @endpoint = endpoint
+    build_client_pool(5)
+  end
+
+  #
+  # POST data and don't wait for the endpoint to process the data before getting a response
+  #
+  def post_data_async(data_hash, path)
+    post_data(data_hash.merge(EXEC_ASYNC), path)
+  end
+
+  #
+  # POST data to the HTTP endpoint
+  #
+  # @param data_hash - A hash representation of the object to be posted. Cannot be nil or empty.
+  # @param path - The URI path to post to
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def post_data(data_hash, path)
+    begin
+      raise 'Data to post to remote service cannot be null or empty' if (data_hash.nil? or data_hash.empty?)
+
+      client =  @client_pool.pop()
+      request_opts = build_request_opts(POST_REQUEST, data_hash, path)
+      request = client.request_raw(request_opts)
+      response = client._send_recv(request)
+
+      if response.code == 200
+        #puts "POST request: #{path} with body: #{json_body} sent successfully"
+        return SuccessResponse.new(response)
+      else
+        puts "POST request: #{path} with body: #{json_body} failed with code: #{response.code} message: #{response.body}"
+        return FailedResponse.new(response)
+      end
+    ensure
+      @client_pool << client
+    end
+  end
+
+  #
+  # GET data from the HTTP endpoint
+  #
+  # @param data_hash - A hash representation of the object to be posted. Cann be nil or empty.
+  # @param path - The URI path to post to
+  #
+  # @return A wrapped response (ResponseWrapper), see below.
+  #
+  def get_data(data_hash, path)
+    begin
+      client =  @client_pool.pop()
+      request_opts = build_request_opts(GET_REQUEST, data_hash, path)
+      request = client.request_raw(request_opts)
+      response = client._send_recv(request)
+
+      if (response.code == 200)
+        # puts 'request sent successfully'
+        return SuccessResponse.new(response)
+      else
+        puts "GET request: #{path} failed with code: #{response.code} message: #{response.body}"
+        return FailedResponse.new(response)
+      end
+    ensure
+      @client_pool << client
+    end
+  end
+
+  #
+  # TODO: fix this
+  #
+  def active
+    return true
+  end
+
+  # def do_nl_search(search)
+  #   search_item = search.query.split(".")[0]
+  #   case search_item
+  #     when "host"
+  #       do_host_search(search)
+  #   end
+  # end
+
+  # def active
+  #   begin
+  #     request_opts = {'method' => 'GET', 'uri' => ONLINE_TEST_URL}
+  #     request = @client.request_raw(request_opts)
+  #     response = @client._send_recv(request)
+  #     if response.code == 200
+  #       try_sound_effect()
+  #       return true
+  #     else
+  #       puts "request failed with code: #{response.code} message: #{response.message}"
+  #       return false
+  #     end
+  #   rescue Exception => e
+  #     puts "Unable to contact goliath service: #{e.message}"
+  #     return false
+  #   end
+  # end
+
+  def name
+    "remote_data_service: (#{@endpoint})"
+  end
+
+  def set_header(key, value)
+    if (@headers.nil?)
+      @headers = Hash.new()
+    end
+
+    @headers[key] = value
+  end
+
+  #########
+  protected
+  #########
+
+  #
+  # Simple response wrapper
+  #
+  class ResponseWrapper
+    attr_reader :response
+    attr_reader :expected
+
+    def initialize(response, expected)
+      @response = response
+      @expected = expected
+    end
+  end
+
+  #
+  # Failed response wrapper
+  #
+  class FailedResponse < ResponseWrapper
+    def initialize(response)
+      super(response, false)
+    end
+  end
+
+  #
+  # Success response wrapper
+  #
+  class SuccessResponse < ResponseWrapper
+    def initialize(response)
+      super(response, true)
+    end
+  end
+
+  #######
+  private
+  #######
+
+  def validate_endpoint(endpoint)
+    raise 'Endpoint cannot be nil' if endpoint.nil?
+    raise "Endpoint: #{endpoint.class} not of type RemoteServiceEndpoint" unless endpoint.is_a?(RemoteServiceEndpoint)
+  end
+
+  def append_workspace(data_hash)
+    workspace = data_hash[:workspace]
+    unless (workspace)
+      workspace = data_hash.delete(:wspace)
+    end
+
+    if (workspace and workspace.is_a?(OpenStruct))
+      data_hash['workspace'] = workspace.name
+    end
+
+    if (workspace.nil?)
+      data_hash['workspace'] = current_workspace_name
+    end
+
+    data_hash
+  end
+
+  def build_request_opts(request_type, data_hash, path)
+    request_opts = {
+        'method' => request_type,
+        'ctype' => 'application/json',
+        'uri' => path}
+
+    if (not data_hash.nil? and not data_hash.empty?)
+      json_body = append_workspace(data_hash).to_json
+      request_opts['data'] = json_body
+    end
+
+    if (not @headers.nil? and not @header.empty?)
+      request_opts['headers'] = @headers
+    end
+
+    request_opts
+  end
+
+  def build_client_pool(size)
+    @client_pool = Queue.new()
+    (1..size).each {
+      @client_pool << Rex::Proto::Http::Client.new(
+          @endpoint.host,
+          @endpoint.port,
+          {},
+          @endpoint.use_ssl,
+          @endpoint.ssl_version)
+    }
+  end
+
+  def try_sound_effect()
+    sound_file = ::File.join(Msf::Config.data_directory, "sounds", "Goliath_Online_Sound_Effect.wav")
+    Rex::Compat.play_sound(sound_file)
+  end
+
+end
+end
+end
+end
+

lib/metasploit/framework/data_service/remote/http/data_service_auto_loader.rb

@@ -0,0 +1,19 @@
+#
+# Autoloads specific remote data services
+#
+module DataServiceAutoLoader
+  autoload :RemoteHostDataService, 'metasploit/framework/data_service/remote/http/remote_host_data_service'
+  autoload :RemoteEventDataService, 'metasploit/framework/data_service/remote/http/remote_event_data_service'
+  autoload :RemoteNoteDataService, 'metasploit/framework/data_service/remote/http/remote_note_data_service'
+  autoload :RemoteWorkspaceDataService, 'metasploit/framework/data_service/remote/http/remote_workspace_data_service'
+  autoload :RemoteVulnDataService, 'metasploit/framework/data_service/remote/http/remote_vuln_data_service'
+  autoload :RemoteWebDataService, 'metasploit/framework/data_service/remote/http/remote_web_data_service'
+  autoload :RemoteServiceDataService, 'metasploit/framework/data_service/remote/http/remote_service_data_service'
+  include RemoteHostDataService
+  include RemoteEventDataService
+  include RemoteNoteDataService
+  include RemoteWorkspaceDataService
+  include RemoteVulnDataService
+  include RemoteWebDataService
+  include RemoteServiceDataService
+end

lib/metasploit/framework/data_service/remote/http/query_meta.rb

@@ -0,0 +1,18 @@
+class QueryMeta
+  attr_accessor :filter_on
+  attr_accessor :associated_attributes
+
+  def initialize
+    @filter_on = []
+    @associated_attributes = []
+  end
+
+  def add_filter_item(item)
+    @filter_on << item
+  end
+
+  def add_associated_attribute(query_meta)
+    @associated_attributes << query_meta
+  end
+
+end

lib/metasploit/framework/data_service/remote/http/remote_event_data_service.rb

@@ -0,0 +1,7 @@
+module RemoteEventDataService
+  EVENT_API_PATH = '/api/1/msf/event'
+
+  def report_event(opts)
+    self.post_data_async(opts, EVENT_API_PATH)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_host_data_service.rb

@@ -0,0 +1,29 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteHostDataService
+  include ResponseDataHelper
+
+  HOST_PATH = '/api/1/msf/host'
+  HOST_SEARCH_PATH = HOST_PATH + "/search"
+
+  def hosts(opts)
+    json_to_open_struct_object(self.get_data(opts, HOST_PATH), [])
+  end
+
+  def report_host(opts)
+    self.post_data_async(opts, HOST_PATH)
+  end
+
+  def find_or_create_host(opts)
+    json_to_open_struct_object(self.post_data(host, HOST_PATH))
+  end
+
+  def report_hosts(hosts)
+    self.post_data(hosts, HOST_PATH)
+  end
+
+  def do_host_search(search)
+    response = self.post_data(search, HOST_SEARCH_PATH)
+    return response.body
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_note_data_service.rb

@@ -0,0 +1,11 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteNoteDataService
+  include ResponseDataHelper
+
+  NOTE_API_PATH = '/api/1/msf/note'
+
+  def report_note(opts)
+    self.post_data_async(opts, NOTE_API_PATH)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_service_data_service.rb

@@ -0,0 +1,7 @@
+module RemoteServiceDataService
+  SERVICE_API_PATH = '/api/1/msf/service'
+
+  def report_service(opts)
+    self.post_data_async(opts, SERVICE_API_PATH)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_service_endpoint.rb

@@ -0,0 +1,26 @@
+module Metasploit
+module Framework
+module DataService
+class RemoteServiceEndpoint
+
+  attr_reader :host
+  attr_reader :port
+  attr_reader :use_ssl
+  attr_reader :ssl_version
+
+  def initialize (host, port = 80, use_ssl = false, ssl_version = 'TLS1')
+    raise 'host cannot be null' if host.nil?
+
+    @host = host
+    @port = port
+    @use_ssl = use_ssl
+    @ssl_version = use_ssl ? ssl_version : nil
+  end
+
+  def to_s
+    "host: #{@host}, port: #{@port}"
+  end
+end
+end
+end
+end

lib/metasploit/framework/data_service/remote/http/remote_vuln_data_service.rb

@@ -0,0 +1,7 @@
+module RemoteVulnDataService
+
+  VULN_API_PATH = '/api/1/msf/vuln'
+  def report_vuln(opts)
+    self.post_data_async(opts, VULN_API_PATH)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_web_data_service.rb

@@ -0,0 +1,11 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteWebDataService
+  include ResponseDataHelper
+
+  WEB_API_PATH = '/api/1/msf/web'
+
+  def report_web_site(opts)
+    self.post_data_async(opts, WEB_API_PATH)
+  end
+end

lib/metasploit/framework/data_service/remote/http/remote_workspace_data_service.rb

@@ -0,0 +1,46 @@
+require 'metasploit/framework/data_service/remote/http/response_data_helper'
+
+module RemoteWorkspaceDataService
+  include ResponseDataHelper
+
+  WORKSPACE_API_PATH = '/api/1/msf/workspace'
+  DEFAULT_WORKSPACE_NAME = 'default'
+
+  def find_workspace(workspace_name)
+    workspace = workspace_cache[workspace_name]
+    return workspace unless (workspace.nil?)
+
+    workspace = json_to_open_struct_object(self.get_data({:workspace_name => workspace_name}, WORKSPACE_API_PATH))
+    workspace_cache[workspace_name] = workspace
+  end
+
+  def add_workspace(workspace_name)
+    self.post_data({:workspace_name => workspace_name}, WORKSPACE_API_PATH)
+  end
+
+  def default_workspace
+    find_workspace(DEFAULT_WORKSPACE_NAME)
+  end
+
+  def workspace
+    find_workspace(current_workspace_name)
+  end
+
+  def workspace=(workspace)
+    @current_workspace_name = workspace.name
+  end
+
+  def workspaces
+    json_to_open_struct_object(self.get_data({:all => true}, WORKSPACE_API_PATH), [])
+  end
+
+  protected
+
+  def workspace_cache
+    @workspace_cache ||= {}
+  end
+
+  def current_workspace_name
+    @current_workspace_name ||= DEFAULT_WORKSPACE_NAME
+  end
+end

lib/metasploit/framework/data_service/remote/http/response_data_helper.rb

@@ -0,0 +1,33 @@
+require 'ostruct'
+
+#
+# HTTP response helper class
+#
+module ResponseDataHelper
+
+  #
+  # Converts an HTTP response to an OpenStruct object
+  #
+  def json_to_open_struct_object(response_wrapper, returns_on_error = nil)
+    if (response_wrapper.expected)
+      begin
+        body = response_wrapper.response.body
+        if (not body.nil? and not body.empty?)
+          return JSON.parse(body, object_class: OpenStruct)
+        end
+      rescue Exception => e
+        puts "open struct conversion failed #{e.message}"
+      end
+    end
+
+    return returns_on_error
+  end
+
+  #
+  # Converts a hash to an open struct
+  #
+  def open_struct(hash)
+    OpenStruct.new(hash)
+  end
+
+end

lib/metasploit/framework/data_service/stubs/event_data_service.rb

@@ -0,0 +1,7 @@
+module EventDataService
+
+  def report_event(opts)
+    raise 'EventDataService#report_event is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/host_data_service.rb

@@ -0,0 +1,18 @@
+module HostDataService
+
+  def report_host(opts)
+    raise 'HostDataService#report_host is not implemented'
+  end
+
+  def report_hosts(hosts)
+    raise 'HostDataService#report_hosts is not implemented'
+  end
+
+  def hosts(opts)
+    raise 'HostDataService#hosts is not implemented'
+  end
+
+  def find_or_create_host(opts)
+    raise 'HostDataService#find_or_create_host is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/note_data_service.rb

@@ -0,0 +1,7 @@
+module NoteDataService
+
+  def report_note(opts)
+    raise 'NoteDataService#report_note is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/query_service.rb

@@ -0,0 +1,7 @@
+module QueryService
+
+  def do_nl_search(search)
+    raise 'QueryService#do_nl_search is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/search.rb

@@ -0,0 +1,4 @@
+class Search
+  attr_accessor :query
+  attr_accessor :projections
+end

lib/metasploit/framework/data_service/stubs/service_data_service.rb

@@ -0,0 +1,7 @@
+module ServiceDataService
+
+  def report_service(opts)
+    raise 'ServiceDataService#report_service is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/vuln_data_service.rb

@@ -0,0 +1,7 @@
+module VulnDataService
+
+  def report_vuln(opts)
+    raise 'VulnDataServicee#report_vuln is not implemented'
+  end
+
+end

lib/metasploit/framework/data_service/stubs/web_data_service.rb

@@ -0,0 +1,6 @@
+module WebDataService
+
+  def report_web_site(opts)
+    raise 'WebDataService#report_web_site is not implemented'
+  end
+end

lib/metasploit/framework/data_service/stubs/workspace_data_service.rb

@@ -0,0 +1,27 @@
+module WorkspaceDataService
+
+  def find_workspace(workspace_name)
+    raise 'WorkspaceDataService#find_workspace is not implemented'
+  end
+
+  def add_workspace(workspace_name)
+    raise 'WorkspaceDataService#add_workspace is not implemented'
+  end
+
+  def default_workspace
+    raise 'WorkspaceDataService#add_workspace is not implemented'
+  end
+
+  def workspace
+    raise 'WorkspaceDataService#add_workspace is not implemented'
+  end
+
+  def workspace=(workspace)
+    raise 'WorkspaceDataService#workspace= is not implemented'
+  end
+
+  def workspaces
+    raise 'WorkspaceDataService#add_workspace is not implemented'
+  end
+
+end

lib/metasploit/framework/parsed_options/base.rb

@@ -137,6 +137,10 @@ class Metasploit::Framework::ParsedOptions::Base
         options.database.config = path
       end
 
+      option_parser.on('-dbrp', 'Run database as a remote process') do
+        options.database.remote_process = true
+      end
+
       option_parser.separator ''
       option_parser.separator 'Framework options'
 

lib/metasploit/framework/parsed_options/remote_db.rb

@@ -0,0 +1,44 @@
+require 'metasploit/framework/parsed_options/base'
+
+class Metasploit::Framework::ParsedOptions::RemoteDB < Metasploit::Framework::ParsedOptions::Base
+
+  def options
+    unless @options
+      super.tap { |options|
+        options.console = ActiveSupport::OrderedOptions.new
+
+        options.console.commands = []
+        options.console.confirm_exit = false
+        options.console.histfile = nil
+        options.console.local_output = nil
+        options.console.plugins = []
+        options.console.quiet = false
+        options.console.real_readline = false
+        options.console.resources = []
+        options.console.subcommand = :run
+      }
+    end
+
+    @options
+  end
+
+  private
+
+  def option_parser
+    unless @option_parser
+      super.tap { |option_parser|
+        option_parser.banner = "Usage: #{option_parser.program_name} [options]"
+
+        option_parser.separator ''
+        option_parser.separator 'Remote DB options:'
+
+        option_parser.on('-ns', "Remove signal processing") do
+          options.database.no_signal = true
+        end
+
+      }
+    end
+
+    @option_parser
+  end
+end

lib/msf/base/simple/framework/module_paths.rb

@@ -12,20 +12,28 @@ module Msf
             return
           end
 
+          #is_remote_db = opts.delete(:is_remote_database)
           allowed_module_paths = []
-          extract_engine_module_paths(Rails.application).each do |path|
-            allowed_module_paths << path
-          end
+
+          #if (!is_remote_db)
+            extract_engine_module_paths(Rails.application).each do |path|
+              allowed_module_paths << path
+            end
+          #else
+          #  allowed_module_paths << "/home/chlee/rapid7/metasploit-framework/modules"
+          #end
 
           if Msf::Config.user_module_directory
             allowed_module_paths << Msf::Config.user_module_directory
           end
 
-          ::Rails::Engine.subclasses.map(&:instance).each do |engine|
-            extract_engine_module_paths(engine).each do |path|
-              allowed_module_paths << path
+          #unless (is_remote_db)
+            ::Rails::Engine.subclasses.map(&:instance).each do |engine|
+              extract_engine_module_paths(engine).each do |path|
+                allowed_module_paths << path
+              end
             end
-          end
+         # end
 
           # If additional module paths have been defined globally, then load them.
           # They should be separated by semi-colons.

lib/msf/core/db_manager.rb

@@ -9,7 +9,6 @@ require 'rex/socket'
 #
 # Project
 #
-
 require 'metasploit/framework/require'
 require 'msf/base/config'
 require 'msf/core'
@@ -17,6 +16,8 @@ require 'msf/core/database_event'
 require 'msf/core/db_import_error'
 require 'msf/core/host_state'
 require 'msf/core/service_state'
+require 'metasploit/framework/data_service'
+
 
 # The db module provides persistent storage and events. This class should be instantiated LAST
 # as the active_suppport library overrides Kernel.require, slowing down all future code loads.
@@ -60,6 +61,9 @@ class Msf::DBManager
 
   optionally_include_metasploit_credential_creation
 
+  # Interface must be included first
+  include Metasploit::Framework::DataService
+
   include Msf::DBManager::Adapter
   include Msf::DBManager::Client
   include Msf::DBManager::Connection
@@ -90,9 +94,14 @@ class Msf::DBManager
   include Msf::DBManager::Web
   include Msf::DBManager::Workspace
 
+
   # Provides :framework and other accessors
   include Msf::Framework::Offspring
 
+  def name
+    'local_db_service'
+  end
+
   #
   # Attributes
   #
@@ -122,7 +131,9 @@ class Msf::DBManager
       return
     end
 
-    initialize_database_support
+    init_success = initialize_database_support
+
+    return init_success
   end
 
   #
@@ -163,4 +174,61 @@ class Msf::DBManager
 
     true
   end
+
+  def init_db(opts)
+
+    init_success = false
+
+    # Append any migration paths necessary to bring the database online
+    if opts['DatabaseMigrationPaths']
+      opts['DatabaseMigrationPaths'].each do |migrations_path|
+        ActiveRecord::Migrator.migrations_paths << migrations_path
+      end
+    end
+
+    if connection_established?
+      after_establish_connection
+    else
+      configuration_pathname = Metasploit::Framework::Database.configurations_pathname(path: opts['DatabaseYAML'])
+
+      unless configuration_pathname.nil?
+        if configuration_pathname.readable?
+          dbinfo = YAML.load_file(configuration_pathname) || {}
+          dbenv  = opts['DatabaseEnv'] || Rails.env
+          db     = dbinfo[dbenv]
+        else
+          elog("Warning, #{configuration_pathname} is not readable. Try running as root or chmod.")
+        end
+
+        if not db
+          elog("No database definition for environment #{dbenv}")
+        else
+          init_success = connect(db)
+        end
+      end
+    end
+
+    # framework.db.active will be true if after_establish_connection ran directly when connection_established? was
+    # already true or if framework.db.connect called after_establish_connection.
+    if !! error
+      if error.to_s =~ /RubyGem version.*pg.*0\.11/i
+        elog("***")
+        elog("*")
+        elog("* Metasploit now requires version 0.11 or higher of the 'pg' gem for database support")
+        elog("* There a three ways to accomplish this upgrade:")
+        elog("* 1. If you run Metasploit with your system ruby, simply upgrade the gem:")
+        elog("*    $ rvmsudo gem install pg ")
+        elog("* 2. Use the Community Edition web interface to apply a Software Update")
+        elog("* 3. Uninstall, download the latest version, and reinstall Metasploit")
+        elog("*")
+        elog("***")
+        elog("")
+        elog("")
+      end
+
+      elog("Failed to connect to the database: #{error}")
+    end
+
+    return init_success
+  end
 end

lib/msf/core/db_manager/event.rb

@@ -8,7 +8,7 @@ module Msf::DBManager::Event
   def report_event(opts = {})
     return if not active
   ::ActiveRecord::Base.connection_pool.with_connection {
-    wspace = opts.delete(:workspace) || workspace
+    wspace = get_workspace(opts)
     return if not wspace # Temp fix?
     uname  = opts.delete(:username)
 

lib/msf/core/db_manager/host.rb

@@ -22,6 +22,9 @@ module Msf::DBManager::Host
 
   # Exactly like report_host but waits for the database to create a host and returns it.
   def find_or_create_host(opts)
+    host = get_host(opts)
+    return host unless host.nil?
+
     report_host(opts)
   end
 
@@ -57,11 +60,16 @@ module Msf::DBManager::Host
   end
 
   # Returns a list of all hosts in the database
-  def hosts(wspace = workspace, only_up = false, addresses = nil)
+  def hosts(opts)
+    wspace = opts[:workspace] || opts[:wspace] || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
   ::ActiveRecord::Base.connection_pool.with_connection {
     conditions = {}
-    conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up
-    conditions[:address] = addresses if addresses
+    conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if opts[:non_dead]
+    conditions[:address] = opts[:addresses] if opts[:addresses]
     wspace.hosts.where(conditions).order(:address)
   }
   end

lib/msf/core/db_manager/http/db_manager_proxy.rb

@@ -0,0 +1,23 @@
+require 'singleton'
+require 'msf/core/db_manager'
+require 'metasploit/framework/database'
+require 'rails'
+
+class DBManagerProxy
+  include Singleton
+
+  attr_reader :db
+
+  private
+
+  def initialize
+    @db = Msf::DBManager.new(self)
+    @db.init_db(parse_opts)
+  end
+
+  def parse_opts
+    opts = {}
+    opts['DatabaseYAML'] = Metasploit::Framework::Database.configurations_pathname.try(:to_path)
+    opts
+  end
+end

lib/msf/core/db_manager/http/http_db_manager_service.rb

@@ -0,0 +1,73 @@
+require 'rack'
+require 'msf/core/db_manager/http/sinatra_app'
+require 'metasploit/framework/parsed_options/remote_db'
+
+class HttpDBManagerService
+
+  def start(opts)
+    parsed_options = Metasploit::Framework::ParsedOptions::RemoteDB.new
+    if (parsed_options.options.database.no_signal)
+      puts 'removing trap'
+      opts[:signals] = false
+      @shutdown_on_interupt = false
+    else
+      @shutdown_on_interupt = true
+    end
+
+    require_environment!(parsed_options)
+
+    init_db
+    start_http_server(opts)
+  end
+
+  private
+
+  def start_http_server(opts)
+
+    Rack::Handler::Thin.run(SinatraApp, opts) do |server|
+
+      # TODO: prevent accidental shutdown from msfconle eg: ctrl-c
+      [:INT, :TERM].each { |sig|
+        trap(sig) {
+          server.stop if (@shutdown_on_interupt || sig == :TERM)
+        }
+      }
+
+      server.threaded = true
+    end
+  end
+
+  def init_db
+    DBManagerProxy.instance
+  end
+
+  def require_environment!(parsed_options)
+    # RAILS_ENV must be set before requiring 'config/application.rb'
+    parsed_options.environment!
+    ARGV.replace(parsed_options.positional)
+
+    # allow other Rails::Applications to use this command
+    if !defined?(Rails) || Rails.application.nil?
+      # @see https://github.com/rails/rails/blob/v3.2.17/railties/lib/rails/commands.rb#L39-L40
+      require Pathname.new(__FILE__).parent.parent.parent.parent.parent.parent.join('config', 'application')
+    end
+
+    # have to configure before requiring environment because
+    # config/environment.rb calls initialize! and the initializers will use
+    # the configuration from the parsed options.
+    parsed_options.configure(Rails.application)
+
+    Rails.application.require_environment!
+  end
+
+  # def init_servlets(http_server)
+  #   servlet_path = File.dirname(__FILE__) + '/servlet/*'
+  #   Dir.glob(servlet_path).collect{|file_path|
+  #     servlet_class = File.basename(file_path, '.rb').classify
+  #     require file_path
+  #     servlet_class_constant = servlet_class.constantize
+  #     http_server.mount servlet_class_constant.api_path, servlet_class_constant
+  #   }
+  # end
+
+end

lib/msf/core/db_manager/http/job_processor.rb

@@ -0,0 +1,39 @@
+require 'singleton'
+
+class JobProcessor
+  include Singleton
+
+  def submit_job(job_args, &job)
+    @job_queue << JobWraper.new(job_args, &job)
+  end
+
+  private
+
+  def initialize
+    @job_queue = Queue.new()
+    start_processor_thread()
+  end
+
+  def start_processor_thread()
+    Thread.new {
+      loop do
+        wrapper = @job_queue.pop()
+        begin
+          wrapper.job.call(wrapper.job_args)
+        rescue Exception => e
+          puts "Error executing job #{e.message}"
+        end
+      end
+    }
+  end
+
+  class JobWraper
+    attr_reader :job
+    attr_reader :job_args
+
+    def initialize(job_args, &job)
+      @job_args = job_args
+      @job = job
+    end
+  end
+end

lib/msf/core/db_manager/http/servlet/event_servlet.rb

@@ -0,0 +1,19 @@
+module EventServlet
+
+  def self.api_path
+    '/api/1/msf/event'
+  end
+
+  def self.registered(app)
+    app.post EventServlet.api_path, &report_event
+  end
+
+  private
+
+  def self.report_event
+    lambda {
+      job = lambda { |opts| get_db().report_event(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/host_servlet.rb

@@ -0,0 +1,32 @@
+module HostServlet
+
+  def self.api_path
+    '/api/1/msf/host'
+  end
+
+  def self.registered(app)
+    app.get HostServlet.api_path, &get_host
+    app.post HostServlet.api_path, &report_host
+  end
+
+  private
+
+  def self.get_host
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().hosts(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_host
+    lambda {
+        job = lambda { |opts| get_db().report_host(opts) }
+        exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/note_servlet.rb

@@ -0,0 +1,20 @@
+module NoteServlet
+
+  def self.api_path
+    '/api/1/msf/note'
+  end
+
+  def self.registered(app)
+    app.post NoteServlet.api_path, &report_note
+  end
+
+  private
+
+  def self.report_note
+    lambda {
+        job = lambda { |opts|  get_db().report_note(opts) }
+        exec_report_job(request, &job)
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/online_test_servlet.rb

@@ -0,0 +1,19 @@
+module OnlineTestServlet
+
+  def self.api_path
+    '/api/1/msf/online'
+  end
+
+  def self.registered(app)
+    app.get OnlineTestServlet.api_path, &get_active
+  end
+
+  private
+
+  def self.get_active
+    lambda {
+      set_empty_response()
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/service_servlet.rb

@@ -0,0 +1,31 @@
+module ServiceServlet
+
+  def self.api_path
+    '/api/1/msf/service'
+  end
+
+  def self.registered(app)
+    app.post ServiceServlet.api_path, &report_service
+  end
+
+  private
+
+  def self.get_host
+    lambda {
+      begin
+        opts = parse_json_request(request, false)
+        data = get_db().hosts(opts)
+        set_json_response(data)
+      rescue Exception => e
+        set_error_on_response(e)
+      end
+    }
+  end
+
+  def self.report_service
+    lambda {
+      job = lambda { |opts| get_db().report_service(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+end

lib/msf/core/db_manager/http/servlet/vuln_servlet.rb

@@ -0,0 +1,23 @@
+module VulnServlet
+
+  def self.api_path
+    '/api/1/msf/vuln'
+  end
+
+  def self.registered(app)
+    app.post VulnServlet.api_path, &report_vuln
+  end
+
+
+  private
+
+  def self.report_vuln
+    lambda {
+      job = lambda { |opts|
+        get_db().report_vuln(opts)
+      }
+      exec_report_job(request, &job)
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/web_servlet.rb

@@ -0,0 +1,20 @@
+module WebServlet
+
+  def self.api_path
+    '/api/1/msf/web'
+  end
+
+  def self.registered(app)
+    app.post WebServlet.api_path, &report_web
+  end
+
+  private
+
+  def self.report_web
+    lambda {
+      job = lambda { |opts|  get_db().report_web_site(opts) }
+      exec_report_job(request, &job)
+    }
+  end
+
+end

lib/msf/core/db_manager/http/servlet/workspace_servlet.rb

@@ -0,0 +1,42 @@
+module WorkspaceServlet
+
+    def self.api_path
+      '/api/1/msf/workspace'
+    end
+
+    def self.registered(app)
+      app.get WorkspaceServlet.api_path, &get_workspace
+      app.post WorkspaceServlet.api_path, &add_workspace
+    end
+
+    private
+
+    def self.get_workspace
+      lambda {
+        begin
+          opts = parse_json_request(request, true)
+          if (opts[:all])
+            data = get_db().workspaces
+          else
+            data = get_db().find_workspace(opts[:workspace_name])
+          end
+
+          set_json_response(data)
+        rescue Exception => e
+          set_error_on_response(e)
+        end
+      }
+    end
+
+    def self.add_workspace
+      lambda {
+        begin
+          opts = parse_json_request(request, true)
+          get_db().add_workspace(opts[:workspace_name])
+          set_empty_response
+        rescue Exception => e
+          set_error_on_response(e)
+        end
+      }
+    end
+end

lib/msf/core/db_manager/http/servlet_helper.rb

@@ -0,0 +1,140 @@
+require 'json'
+require 'msf/core/db_manager/http/db_manager_proxy'
+require 'msf/core/db_manager/http/job_processor'
+
+module ServletHelper
+
+  def set_error_on_response(error)
+    puts "Error handling request: #{error.message}"
+    headers = {'Content-Type' => 'text/plain'}
+    [500, headers, error.message]
+  end
+
+  def set_empty_response()
+    [200,  '']
+  end
+
+  def set_json_response(data)
+    headers = {'Content-Type' => 'application/json'}
+    [200, headers, to_json(data)]
+  end
+
+  def parse_json_request(request, strict = false)
+    body = request.body.read
+    if (body.nil? || body.empty?)
+      raise 'Invalid body, expected data' if strict
+      return {}
+    end
+
+    hash = JSON.parse(body)
+    hash.symbolize_keys
+  end
+
+  def exec_report_job(request, &job)
+    begin
+
+      # report jobs always need data
+      opts = parse_json_request(request, true)
+
+      exec_async = opts.delete(:exec_async)
+      if (exec_async)
+        JobProcessor.instance.submit_job(opts, &job)
+      else
+        data = job.call(opts)
+        set_json_response(data)
+      end
+
+    rescue Exception => e
+      set_error_on_response(e)
+    end
+  end
+
+  def get_db()
+    DBManagerProxy.instance.db
+  end
+
+  #######
+  private
+  #######
+
+  def to_json(data)
+    return '{}' if data.nil?
+    json = data.to_json
+    return json.to_s
+  end
+
+
+  # TODO: add query meta
+  # Returns a hash representing the model. Some configuration can be
+  # passed through +options+.
+  #
+  # The option <tt>include_root_in_json</tt> controls the top-level behavior
+  # of +as_json+. If +true+, +as_json+ will emit a single root node named
+  # after the object's type. The default value for <tt>include_root_in_json</tt>
+  # option is +false+.
+  #
+  #   user = User.find(1)
+  #   user.as_json
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #     "created_at" => "2006/08/01", "awesome" => true}
+  #
+  #   ActiveRecord::Base.include_root_in_json = true
+  #
+  #   user.as_json
+  #   # => { "user" => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #                  "created_at" => "2006/08/01", "awesome" => true } }
+  #
+  # This behavior can also be achieved by setting the <tt>:root</tt> option
+  # to +true+ as in:
+  #
+  #   user = User.find(1)
+  #   user.as_json(root: true)
+  #   # => { "user" => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #                  "created_at" => "2006/08/01", "awesome" => true } }
+  #
+  # Without any +options+, the returned Hash will include all the model's
+  # attributes.
+  #
+  #   user = User.find(1)
+  #   user.as_json
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true}
+  #
+  # The <tt>:only</tt> and <tt>:except</tt> options can be used to limit
+  # the attributes included, and work similar to the +attributes+ method.
+  #
+  #   user.as_json(only: [:id, :name])
+  #   # => { "id" => 1, "name" => "Konata Izumi" }
+  #
+  #   user.as_json(except: [:id, :created_at, :age])
+  #   # => { "name" => "Konata Izumi", "awesome" => true }
+  #
+  # To include the result of some method calls on the model use <tt>:methods</tt>:
+  #
+  #   user.as_json(methods: :permalink)
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "permalink" => "1-konata-izumi" }
+  #
+  # To include associations use <tt>:include</tt>:
+  #
+  #   user.as_json(include: :posts)
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "posts" => [ { "id" => 1, "author_id" => 1, "title" => "Welcome to the weblog" },
+  #   #                   { "id" => 2, "author_id" => 1, "title" => "So I was thinking" } ] }
+  #
+  # Second level and higher order associations work as well:
+  #
+  #   user.as_json(include: { posts: {
+  #                              include: { comments: {
+  #                                             only: :body } },
+  #                              only: :title } })
+  #   # => { "id" => 1, "name" => "Konata Izumi", "age" => 16,
+  #   #      "created_at" => "2006/08/01", "awesome" => true,
+  #   #      "posts" => [ { "comments" => [ { "body" => "1st post!" }, { "body" => "Second!" } ],
+  #   #                     "title" => "Welcome to the weblog" },
+  #   #                   { "comments" => [ { "body" => "Don't think too hard" } ],
+  #   #
+
+end

lib/msf/core/db_manager/http/sinatra_app.rb

@@ -0,0 +1,26 @@
+require 'sinatra/base'
+require 'msf/core/db_manager/http/servlet_helper'
+require 'msf/core/db_manager/http/servlet/host_servlet'
+require 'msf/core/db_manager/http/servlet/note_servlet'
+require 'msf/core/db_manager/http/servlet/vuln_servlet'
+require 'msf/core/db_manager/http/servlet/event_servlet'
+require 'msf/core/db_manager/http/servlet/web_servlet'
+require 'msf/core/db_manager/http/servlet/online_test_servlet'
+require 'msf/core/db_manager/http/servlet/workspace_servlet'
+require 'msf/core/db_manager/http/servlet/service_servlet'
+
+class SinatraApp < Sinatra::Base
+
+  helpers ServletHelper
+
+  # Servlet registration
+  register HostServlet
+  register VulnServlet
+  register EventServlet
+  register WebServlet
+  register OnlineTestServlet
+  register NoteServlet
+  register WorkspaceServlet
+  register ServiceServlet
+
+end

lib/msf/core/db_manager/vuln.rb

@@ -105,6 +105,8 @@ module Msf::DBManager::Vuln
       opts[:refs].each do |r|
         if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val))
           r = "#{r.ctx_id}-#{r.ctx_val}"
+        elsif (r.is_a?(Hash) and r['ctx_id'] and r['ctx_val'])
+          r = "#{r[:ctx_id]}-#{r[:ctx_val]}"
         end
         rids << find_or_create_ref(:name => r)
       end

lib/msf/core/db_manager/workspace.rb

@@ -33,4 +33,9 @@ module Msf::DBManager::Workspace
     ::Mdm::Workspace.order('updated_at asc').load
   }
   end
+
+  def get_workspace(opts)
+    workspace = opts.delete(:wspace) || opts.delete(:workspace) || workspace
+    find_workspace(workspace) if (workspace.is_a?(String))
+  end
 end

lib/msf/core/framework.rb

@@ -58,7 +58,7 @@ class Framework
   require 'msf/core/module_manager'
   require 'msf/core/session_manager'
   require 'msf/core/plugin_manager'
-  require 'msf/core/db_manager'
+  require 'metasploit/framework/data_service/proxy/core'
   require 'msf/core/event_dispatcher'
   require 'rex/json_hash_file'
 
@@ -189,13 +189,13 @@ class Framework
   #
   attr_reader   :browser_profiles
 
-  # The framework instance's db manager. The db manager
-  # maintains the database db and handles db events
   #
-  # @return [Msf::DBManager]
+  # The framework instance's data service proxy
+  #
+  # @return [Metasploit::Framework::DataService::DataProxy]
   def db
     synchronize {
-      @db ||= Msf::DBManager.new(self, options)
+      @db ||= Metasploit::Framework::DataService::DataProxy.instance
     }
   end
 
@@ -302,19 +302,19 @@ class FrameworkEventSubscriber
 
   include ::Msf::UiEventSubscriber
   ##
-  # :category: ::Msf::UiEventSubscriber implementors
+  # :category: ::Msf::Uips -ef | EventSubscriber implementors
   def on_ui_command(command)
-    if framework.db.active
-      report_event(:name => "ui_command", :info => {:command => command})
-    end
+    # if framework.db.active
+    #   report_event(:name => "ui_command", :info => {:command => command})
+    # end
   end
 
   ##
   # :category: ::Msf::UiEventSubscriber implementors
   def on_ui_stop()
-    if framework.db.active
-      report_event(:name => "ui_stop")
-    end
+    # if framework.db.active
+    #   report_event(:name => "ui_stop")
+    # end
   end
 
   ##

lib/msf/core/module_manager/cache.rb

@@ -131,7 +131,8 @@ module Msf::ModuleManager::Cache
   # @return [true] if migrations have been run
   # @return [false] otherwise
   def framework_migrated?
-    framework.db && framework.db.migrated
+    return false
+    #framework.db && framework.db.migrated
   end
 
   # @!attribute [rw] module_info_by_path

lib/msf/core/session_manager.rb

@@ -126,21 +126,21 @@ class SessionManager < Hash
         #
         # Skip the database cleanup code below if there is no database
         #
-        next if not (framework.db and framework.db.active)
+        #next if not (framework.db and framework.db.active)
 
         #
         # Clean out any stale sessions that have been orphaned by a dead
         # framework instance.
         #
-        ::ActiveRecord::Base.connection_pool.with_connection do |conn|
-          ::Mdm::Session.where(closed_at: nil).each do |db_session|
-            if db_session.last_seen.nil? or ((Time.now.utc - db_session.last_seen) > (2*LAST_SEEN_INTERVAL))
-              db_session.closed_at    = db_session.last_seen || Time.now.utc
-              db_session.close_reason = "Orphaned"
-              db_session.save
-            end
-          end
-        end
+        # ::ActiveRecord::Base.connection_pool.with_connection do |conn|
+        #   ::Mdm::Session.where(closed_at: nil).each do |db_session|
+        #     if db_session.last_seen.nil? or ((Time.now.utc - db_session.last_seen) > (2*LAST_SEEN_INTERVAL))
+        #       db_session.closed_at    = db_session.last_seen || Time.now.utc
+        #       db_session.close_reason = "Orphaned"
+        #       db_session.save
+        #     end
+        #   end
+        # end
       end
 
       #

lib/msf/core/thread_manager.rb

@@ -107,16 +107,16 @@ class ThreadManager < Array
           )
           elog("Call Stack\n#{e.backtrace.join("\n")}")
           raise e
-        ensure
-          if framework.db and framework.db.active
-            # NOTE: despite the Deprecation Warning's advice, this should *NOT*
-            # be ActiveRecord::Base.connection.close which causes unrelated
-            # threads to raise ActiveRecord::StatementInvalid exceptions at
-            # some point in the future, presumably due to the pool manager
-            # believing that the connection is still usable and handing it out
-            # to another thread.
-            ::ActiveRecord::Base.connection_pool.release_connection
-          end
+        # ensure
+        #   if framework.db and framework.db.active
+        #     # NOTE: despite the Deprecation Warning's advice, this should *NOT*
+        #     # be ActiveRecord::Base.connection.close which causes unrelated
+        #     # threads to raise ActiveRecord::StatementInvalid exceptions at
+        #     # some point in the future, presumably due to the pool manager
+        #     # believing that the connection is still usable and handing it out
+        #     # to another thread.
+        #     ::ActiveRecord::Base.connection_pool.release_connection
+        #   end
         end
       end
     else

lib/msf/ui/console/command_dispatcher/core.rb

@@ -461,6 +461,7 @@ class Core
     forced = false
     forced = true if (args[0] and args[0] =~ /-y/i)
 
+    framework.db.exit_called
     if(framework.sessions.length > 0 and not forced)
       print_status("You have active sessions open, to exit anyway type \"exit -y\"")
       return

lib/msf/ui/console/command_dispatcher/db.rb

@@ -1,8 +1,12 @@
 # -*- coding: binary -*-
 
+require 'json'
 require 'rexml/document'
 require 'rex/parser/nmap_xml'
 require 'msf/core/db_export'
+require 'metasploit/framework/data_service'
+require 'metasploit/framework/data_service/remote/http/core'
+require 'metasploit/framework/data_service/remote/http/remote_service_endpoint'
 
 module Msf
 module Ui
@@ -42,7 +46,12 @@ class Db
       "db_import"     => "Import a scan result file (filetype will be auto-detected)",
       "db_export"     => "Export a file containing the contents of the database",
       "db_nmap"       => "Executes nmap and records the output automatically",
-      "db_rebuild_cache" => "Rebuilds the database-stored module cache"
+      "db_rebuild_cache" => "Rebuilds the database-stored module cache",
+      "test_data_service_host" => "Blah",
+      "add_goliath_service" => "Blah",
+      "list_data_services" => "Blah",
+      "set_data_service" => "Blah",
+      "nl_search" => "Blah"
     }
 
     # Always include commands that only make sense when connected.
@@ -77,6 +86,32 @@ class Db
     true
   end
 
+  def cmd_set_data_service(service_id)
+    data_service_manager = Metasploit::Framework::DataService::DataProxy.instance
+    data_service_manager.set_data_service(service_id)
+  end
+
+  def cmd_list_data_services()
+    data_service_manager = Metasploit::Framework::DataService::DataProxy..instance
+    data_service_manager.print_data_services
+  end
+
+  def cmd_add_goliath_service(*args)
+      while (arg = args.shift)
+        case arg
+          when '-h'
+            host = args.shift
+          when '-p'
+            port = args.shift
+        end
+      end
+
+      remote_service_endpoint = Metasploit::Framework::DataService::RemoteServiceEndpoint.new(host, port)
+      remote_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(remote_service_endpoint)
+      data_service_manager = Metasploit::Framework::DataService::DataProxy.instance
+      data_service_manager.register_data_service(remote_data_service)
+  end
+
   def cmd_workspace_help
     print_line "Usage:"
     print_line "    workspace                  List workspaces"
@@ -344,7 +379,7 @@ class Db
 
   def cmd_hosts(*args)
     return unless active?
-  ::ActiveRecord::Base.connection_pool.with_connection {
+  #::ActiveRecord::Base.connection_pool.with_connection {
     onlyup = false
     set_rhosts = false
     mode = []
@@ -355,7 +390,34 @@ class Db
     search_term = nil
 
     output = nil
-    default_columns = ::Mdm::Host.column_names.sort
+    default_columns = [
+        'address',
+        'arch',
+        'comm',
+        'comments',
+        'created_at',
+        'cred_count',
+        'detected_arch',
+        'exploit_attempt_count',
+        'host_detail_count',
+        'info',
+        'mac',
+        'name',
+        'note_count',
+        'os_family',
+        'os_flavor',
+        'os_lang',
+        'os_name',
+        'os_sp',
+        'purpose',
+        'scope',
+        'service_count',
+        'state',
+        'updated_at',
+        'virtual_host',
+        'vuln_count',
+        'workspace_id']
+    
     default_columns << 'tags' # Special case
     virtual_columns = [ 'svcs', 'vulns', 'workspace', 'tags' ]
 
@@ -504,8 +566,8 @@ class Db
           # Deal with the special cases
           if virtual_columns.include?(n)
             case n
-            when "svcs";      host.services.length
-            when "vulns";     host.vulns.length
+            when "svcs";      host.service_count
+            when "vulns";     host.vuln_count
             when "workspace"; host.workspace.name
             when "tags"
               found_tags = Mdm::Tag.joins(:hosts).where("hosts.workspace_id = ? and hosts.address = ?", framework.db.workspace.id, host.address).order("tags.id DESC")
@@ -515,7 +577,7 @@ class Db
             end
           # Otherwise, it's just an attribute
           else
-            host.attributes[n] || ""
+            host[n] || ""
           end
         end
 
@@ -546,7 +608,7 @@ class Db
     set_rhosts_from_addrs(rhosts.uniq) if set_rhosts
 
     print_status("Deleted #{delete_count} hosts") if delete_count > 0
-  }
+  #}
   end
 
   def cmd_services_help

lib/msf/ui/console/driver.rb

@@ -123,11 +123,27 @@ class Driver < Msf::Ui::Driver
     end
 
     # Add the database dispatcher if it is usable
-    if (framework.db.usable)
-      require 'msf/ui/console/command_dispatcher/db'
-      enstack_dispatcher(CommandDispatcher::Db)
-      require 'msf/ui/console/command_dispatcher/creds'
-      enstack_dispatcher(CommandDispatcher::Creds)
+    # if (framework.db.usable)
+    #   require 'msf/ui/console/command_dispatcher/db'
+    #   enstack_dispatcher(CommandDispatcher::Db)
+    #   require 'msf/ui/console/command_dispatcher/creds'
+    #   enstack_dispatcher(CommandDispatcher::Creds)
+    # else
+    #   print_error("***")
+    #   if framework.db.error == "disabled"
+    #     print_error("* WARNING: Database support has been disabled")
+    #   else
+    #     print_error("* WARNING: No database support: #{framework.db.error.class} #{framework.db.error}")
+    #   end
+    #   print_error("***")
+    # end
+
+    framework.db.init(framework, opts)
+    if (framework.db.active)
+        require 'msf/ui/console/command_dispatcher/db'
+        enstack_dispatcher(CommandDispatcher::Db)
+        require 'msf/ui/console/command_dispatcher/creds'
+        enstack_dispatcher(CommandDispatcher::Creds)
     else
       print_error("***")
       if framework.db.error == "disabled"
@@ -160,70 +176,70 @@ class Driver < Msf::Ui::Driver
     self.confirm_exit = opts['ConfirmExit']
 
     # Parse any specified database.yml file
-    if framework.db.usable and not opts['SkipDatabaseInit']
-
-      # Append any migration paths necessary to bring the database online
-      if opts['DatabaseMigrationPaths']
-        opts['DatabaseMigrationPaths'].each do |migrations_path|
-          ActiveRecord::Migrator.migrations_paths << migrations_path
-        end
-      end
-
-      if framework.db.connection_established?
-        framework.db.after_establish_connection
-      else
-        configuration_pathname = Metasploit::Framework::Database.configurations_pathname(path: opts['DatabaseYAML'])
-
-        unless configuration_pathname.nil?
-          if configuration_pathname.readable?
-            dbinfo = YAML.load_file(configuration_pathname) || {}
-            dbenv  = opts['DatabaseEnv'] || Rails.env
-            db     = dbinfo[dbenv]
-          else
-            print_error("Warning, #{configuration_pathname} is not readable. Try running as root or chmod.")
-          end
-
-          if not db
-            print_error("No database definition for environment #{dbenv}")
-          else
-            framework.db.connect(db)
-          end
-        end
-      end
-
-      # framework.db.active will be true if after_establish_connection ran directly when connection_established? was
-      # already true or if framework.db.connect called after_establish_connection.
-      if !! framework.db.error
-        if framework.db.error.to_s =~ /RubyGem version.*pg.*0\.11/i
-          print_error("***")
-          print_error("*")
-          print_error("* Metasploit now requires version 0.11 or higher of the 'pg' gem for database support")
-          print_error("* There a three ways to accomplish this upgrade:")
-          print_error("* 1. If you run Metasploit with your system ruby, simply upgrade the gem:")
-          print_error("*    $ rvmsudo gem install pg ")
-          print_error("* 2. Use the Community Edition web interface to apply a Software Update")
-          print_error("* 3. Uninstall, download the latest version, and reinstall Metasploit")
-          print_error("*")
-          print_error("***")
-          print_error("")
-          print_error("")
-        end
-
-        print_error("Failed to connect to the database: #{framework.db.error}")
-      end
-    end
+    # if framework.db.usable and not opts['SkipDatabaseInit']
+    #
+    #   # Append any migration paths necessary to bring the database online
+    #   if opts['DatabaseMigrationPaths']
+    #     opts['DatabaseMigrationPaths'].each do |migrations_path|
+    #       ActiveRecord::Migrator.migrations_paths << migrations_path
+    #     end
+    #   end
+    #
+    #   if framework.db.connection_established?
+    #     framework.db.after_establish_connection
+    #   else
+    #     configuration_pathname = Metasploit::Framework::Database.configurations_pathname(path: opts['DatabaseYAML'])
+    #
+    #     unless configuration_pathname.nil?
+    #       if configuration_pathname.readable?
+    #         dbinfo = YAML.load_file(configuration_pathname) || {}
+    #         dbenv  = opts['DatabaseEnv'] || Rails.env
+    #         db     = dbinfo[dbenv]
+    #       else
+    #         print_error("Warning, #{configuration_pathname} is not readable. Try running as root or chmod.")
+    #       end
+    #
+    #       if not db
+    #         print_error("No database definition for environment #{dbenv}")
+    #       else
+    #         framework.db.connect(db)
+    #       end
+    #     end
+    #   end
+    #
+    #   # framework.db.active will be true if after_establish_connection ran directly when connection_established? was
+    #   # already true or if framework.db.connect called after_establish_connection.
+    #   if !! framework.db.error
+    #     if framework.db.error.to_s =~ /RubyGem version.*pg.*0\.11/i
+    #       print_error("***")
+    #       print_error("*")
+    #       print_error("* Metasploit now requires version 0.11 or higher of the 'pg' gem for database support")
+    #       print_error("* There a three ways to accomplish this upgrade:")
+    #       print_error("* 1. If you run Metasploit with your system ruby, simply upgrade the gem:")
+    #       print_error("*    $ rvmsudo gem install pg ")
+    #       print_error("* 2. Use the Community Edition web interface to apply a Software Update")
+    #       print_error("* 3. Uninstall, download the latest version, and reinstall Metasploit")
+    #       print_error("*")
+    #       print_error("***")
+    #       print_error("")
+    #       print_error("")
+    #     end
+    #
+    #     print_error("Failed to connect to the database: #{framework.db.error}")
+    #   end
+    # end
 
     # Initialize the module paths only if we didn't get passed a Framework instance and 'DeferModuleLoads' is false
     unless opts['Framework'] || opts['DeferModuleLoads']
       # Configure the framework module paths
-      self.framework.init_module_paths(module_paths: opts['ModulePath'])
+      self.framework.init_module_paths(module_paths: opts['ModulePath']) #, is_remote_database: opts['DatabaseRemoteProcess'])
     end
 
-    if framework.db.active && !opts['DeferModuleLoads']
-      framework.threads.spawn("ModuleCacheRebuild", true) do
-        framework.modules.refresh_cache_from_module_files
-      end
-    end
+    # if framework.db.active && !opts['DeferModuleLoads']
+    #   framework.threads.spawn("ModuleCacheRebuild", true) do
+    #     framework.modules.refresh_cache_from_module_files
+    #   end
+    # end
 
     # Load console-specific configuration (after module paths are added)
     load_config(opts['Config'])

msfdb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# -*- coding: binary -*-
+#
+# This user interface provides users with a command console interface to the
+# framework.
+#
+
+#
+# Standard Library
+#
+
+# msfbase = __FILE__
+# while File.symlink?(msfbase)
+#   msfbase = File.expand_path(File.readlink(msfbase), File.dirname(msfbase))
+# end
+#
+# $:.unshift(File.expand_path(File.join(File.dirname(msfbase), 'lib')))
+# require 'msfenv'
+
+require 'pathname'
+
+#
+# Project
+#
+
+# @see https://github.com/rails/rails/blob/v3.2.17/railties/lib/rails/generators/rails/app/templates/script/rails#L3-L5
+require Pathname.new(__FILE__).realpath.expand_path.parent.join('config', 'boot')
+require 'msf/core/db_manager/http/http_db_manager_service'
+HttpDBManagerService.new().start(:Port => '8080', :Host => '127.0.0.1')