Merge branch 'goliath' into MS-2833

lib/metasploit/framework/command/console.rb

@@ -79,7 +79,6 @@ class Metasploit::Framework::Command::Console < Metasploit::Framework::Command::
       driver_options['DatabaseEnv'] = options.environment
       driver_options['DatabaseMigrationPaths'] = options.database.migrations_paths
       driver_options['DatabaseYAML'] = options.database.config
-      driver_options['DatabaseRemoteProcess'] = options.database.remote_process
       driver_options['DeferModuleLoads'] = options.modules.defer_loads
       driver_options['DisableBanner'] = options.console.quiet
       driver_options['DisableDatabase'] = options.database.disable

lib/metasploit/framework/data_service.rb

@@ -33,6 +33,30 @@ module DataService
   def active
     raise 'DataLService#active is not implemented';
   end
+
+  #
+  # Hold metadata about a data service
+  #
+  class Metadata
+    attr_reader :id
+    attr_reader :name
+    attr_reader :active
+
+    def initialize (id, name, active)
+      self.id = id
+      self.name = name
+      self.active = active
+    end
+
+    #######
+    private
+    #######
+
+    attr_writer :id
+    attr_writer :name
+    attr_writer :active
+
+  end
 end
 end
 end

lib/metasploit/framework/data_service/proxy/core.rb

@@ -28,13 +28,13 @@ class DataProxy
   #
   def error
     return @error if (@error)
-    return @data_service.error if @data_service
-    return "none"
+    return @current_data_service.error if @current_data_service
+    return 'none'
   end
 
   def is_local?
-    if (@data_service)
-      return (@data_service.name == 'local_db_service')
+    if (@current_data_service)
+      return (@current_data_service.name == 'local_db_service')
     end
 
     return false
@@ -44,8 +44,8 @@ class DataProxy
   # Determines if the data service is active
   #
   def active
-    if (@data_service)
-      return @data_service.active
+    if (@current_data_service)
+      return @current_data_service.active
     end
 
     return false
@@ -57,8 +57,6 @@ class DataProxy
   #
   def register_data_service(data_service, online=false)
     validate(data_service)
-
-    puts "Registering data service: #{data_service.name}"
     data_service_id = @data_service_id += 1
     @data_services[data_service_id] = data_service
     set_data_service(data_service_id, online)
@@ -70,67 +68,51 @@ class DataProxy
   def set_data_service(data_service_id, online=false)
     data_service = @data_services[data_service_id.to_i]
     if (data_service.nil?)
-      puts "Data service with id: #{data_service_id} does not exist"
-      return nil
+      raise "Data service with id: #{data_service_id} does not exist"
     end
 
     if (!online && !data_service.active)
-      puts "Data service not online: #{data_service.name}, not setting as active"
-      return nil
+      raise "Data service not online: #{data_service.name}, not setting as active"
     end
 
-    puts "Setting active data service: #{data_service.name}"
-    @data_service = data_service
+    @current_data_service = data_service
   end
 
   #
-  # Prints out a list of the current data services
+  # Retrieves metadata about the data services
   #
-  def print_data_services()
+  def get_services_metadata()
+    services_metadata = []
     @data_services.each_key {|key|
-      out = "id: #{key}, description: #{@data_services[key].name}"
-      if (!@data_service.nil? && @data_services[key].name == @data_service.name)
-        out += " [active]"
-      end
-      puts out  #hahaha
+      name = @data_services[key].name
+      active = !@current_data_service.nil? && name == @current_data_service.name
+      services_metadata << Metasploit::Framework::DataService::Metadata.new(key, name, active)
     }
+
+    services_metadata
   end
 
   #
   # Used to bridge the local db
   #
   def method_missing(method, *args, &block)
-    #puts "Attempting to delegate method: #{method}"
-    unless @data_service.nil?
-      @data_service.send(method, *args, &block)
+    dlog ("Attempting to delegate method: #{method}")
+    unless @current_data_service.nil?
+      @current_data_service.send(method, *args, &block)
     end
   end
 
   def respond_to?(method_name, include_private=false)
-    unless @data_service.nil?
-      return @data_service.respond_to?(method_name, include_private)
+    unless @current_data_service.nil?
+      return @current_data_service.respond_to?(method_name, include_private)
     end
 
     false
   end
 
-  #
-  # Attempt to shutdown the local db process if it exists
-  #
-  def exit_called
-    if @pid
-      puts 'Killing db process'
-      begin
-        Process.kill("TERM", @pid)
-      rescue Exception => e
-        puts "Unable to kill db process: #{e.message}"
-      end
-    end
-  end
-
   def get_data_service
-    raise 'No registered data_service' unless @data_service
-    return @data_service
+    raise 'No registered data_service' unless @current_data_service
+    return @current_data_service
   end
 
   #######
@@ -143,14 +125,11 @@ class DataProxy
       if !db_manager.nil?
         register_data_service(db_manager, true)
         @usable = true
-      elsif opts['DatabaseRemoteProcess']
-        run_remote_db_process(opts)
-        @usable = true
       else
         @error = 'disabled'
       end
     rescue Exception => e
-      puts "Unable to initialize a dataservice #{e.message}"
+      raise "Unable to initialize data service: #{e.message}"
     end
   end
 
@@ -170,19 +149,6 @@ class DataProxy
     return false
   end
 
-
-  def run_remote_db_process(opts)
-    # started with no signal to prevent ctrl-c from taking out db
-    db_script = File.join( Msf::Config.install_root, "msfdb -ns")
-    wait_t = Open3.pipeline_start(db_script)
-    @pid = wait_t[0].pid
-    puts "Started process with pid #{@pid}"
-
-    endpoint = URI.parse('http://localhost:8080')
-    remote_host_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint)
-    register_data_service(remote_host_data_service, true)
-  end
-
 end
 end
 end

lib/metasploit/framework/data_service/proxy/credential_data_proxy.rb

@@ -5,7 +5,7 @@ module CredentialDataProxy
       data_service = self.get_data_service()
       data_service.create_credential(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#create_credential threw exception: #{e.message}"
+      elog "Problem creating credential: #{e.message}"
     end
   end
 
@@ -14,8 +14,7 @@ module CredentialDataProxy
       data_service = self.get_data_service
       data_service.creds(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#credentials threw exception: #{e.message}"
-      e.backtrace.each { |line| puts "#{line}\n" }
+      elog "Problem retrieving credentials: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/event_data_proxy.rb

@@ -5,7 +5,7 @@ module EventDataProxy
       data_service = self.get_data_service()
       data_service.report_event(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_event threw exception: #{e.message}"
+      elog "Problem reporting event: #{e.message}"
     end
   end
 

lib/metasploit/framework/data_service/proxy/exploit_data_proxy.rb

@@ -5,7 +5,7 @@ module ExploitDataProxy
       data_service = self.get_data_service()
       data_service.report_exploit_attempt(host, opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_exploit_attempt threw exception: #{e.message}"
+      elog "Problem reporting exploit attempt: #{e.message}"
     end
   end
 
@@ -14,7 +14,7 @@ module ExploitDataProxy
       data_service = self.get_data_service()
       data_service.report_exploit_failure(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_exploit_failure threw exception: #{e.message}"
+      elog "Problem reporting exploit failure: #{e.message}"
     end
   end
 
@@ -23,7 +23,7 @@ module ExploitDataProxy
       data_service = self.get_data_service()
       data_service.report_exploit_success(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_exploit_success threw exception: #{e.message}"
+      elog "Problem reporting exploit success: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/host_data_proxy.rb

@@ -10,14 +10,13 @@ module HostDataProxy
       opts[:search_term] = search_term
       data_service.hosts(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#hosts threw exception: #{e.message}"
+      elog "Problem retrieving hosts: #{e.message}"
     end
   end
 
   # TODO: Shouldn't this proxy to RemoteHostDataService#find_or_create_host ?
   # It's currently skipping the "find" part
   def find_or_create_host(opts)
-    puts 'Calling find host'
     report_host(opts)
   end
 
@@ -28,8 +27,7 @@ module HostDataProxy
       data_service = self.get_data_service()
       data_service.report_host(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#report_host threw exception: #{e.message}"
-      opts.each { |k, v| puts "#{k} : #{v}" }
+      elog "Problem reporting host: #{e.message}"
     end
   end
 
@@ -38,7 +36,7 @@ module HostDataProxy
       data_service = self.get_data_service()
       data_service.report_hosts(hosts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#report_hosts threw exception: #{e.message}"
+      elog "Problem reporting hosts: #{e.message}"
     end
   end
 
@@ -47,7 +45,7 @@ module HostDataProxy
       data_service = self.get_data_service()
       data_service.delete_host(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#delete_host threw exception: #{e.message}"
+      elog "Problem removing host: #{e.message}"
     end
   end
 
@@ -55,13 +53,13 @@ module HostDataProxy
 
   def valid(opts)
     unless opts[:host]
-      puts 'Invalid host hash passed, :host is missing'
+      ilog 'Invalid host hash passed, :host is missing'
       return false
     end
 
     # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
     if opts[:host].eql? "Remote Pipe"
-      puts "Invalid host hash passed, address was of type 'Remote Pipe'"
+      ilog "Invalid host hash passed, address was of type 'Remote Pipe'"
       return false
     end
 

lib/metasploit/framework/data_service/proxy/loot_data_proxy.rb

@@ -8,7 +8,7 @@ module LootDataProxy
       end
       data_service.report_loot(opts)
     rescue Exception => e
-      elog "Problem creating loot: #{e.message}"
+      elog "Problem reporting loot: #{e.message}"
     end
   end
 
@@ -28,6 +28,7 @@ module LootDataProxy
       e.backtrace.each { |line| elog "#{line}\n" }
     end
   end
+
   alias_method :loot, :loots
 
   def update_loot(opts)

lib/metasploit/framework/data_service/proxy/nmap_data_proxy.rb

@@ -5,8 +5,7 @@ module NmapDataProxy
       data_service = self.get_data_service()
       data_service.import_nmap_xml_file(args)
     rescue Exception => e
-      puts "Call to #{data_service.class}#import_nmap_xml_file threw exception: #{e.message}"
-      e.backtrace { |line| puts "#{line}\n"}
+      elog "Problem importing NMAP XML: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/note_data_proxy.rb

@@ -4,7 +4,7 @@ module NoteDataProxy
       data_service = self.get_data_service()
       data_service.report_note(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_note threw exception: #{e.message}"
+      elog "Problem reporting note: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/service_data_proxy.rb

@@ -5,7 +5,7 @@ module ServiceDataProxy
       data_service = self.get_data_service()
       data_service.report_service(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_service threw exception: #{e.message}"
+      elog "Problem reporting service: #{e.message}"
     end
   end
 

lib/metasploit/framework/data_service/proxy/session_data_proxy.rb

@@ -4,8 +4,7 @@ module SessionDataProxy
       data_service = self.get_data_service()
       data_service.report_session(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_session threw exception: #{e.message}"
-      puts e.backtrace.each { |line| puts "#{line}\n" }
+      elog "Problem reporting session: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/session_event_data_proxy.rb

@@ -1,20 +1,11 @@
 module SessionEventDataProxy
 
-  def session_events
-    begin
-      data_service = self.get_data_service()
-      puts "In SessionEventDataProxy.session_events"
-    rescue  Exception => e
-      puts"Call to  #{data_service.class}#session_events threw exception: #{e.message}"
-    end
-  end
-
   def report_session_event(opts)
     begin
       data_service = self.get_data_service()
       data_service.report_session_event(opts)
     rescue Exception => e
-      puts "Call to #{data_service.class}#report_session_event threw exception: #{e.message}"
+      elog "Problem reporting session event: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/vuln_data_proxy.rb

@@ -5,7 +5,7 @@ module VulnDataProxy
       data_service = self.get_data_service()
       data_service.report_vuln(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_vuln threw exception: #{e.message}"
+      elog "Problem reporting vulnerability: #{e.message}"
     end
   end
 

lib/metasploit/framework/data_service/proxy/web_data_proxy.rb

@@ -4,7 +4,7 @@ module WebDataProxy
       data_service = self.get_data_service()
       data_service.report_web_site(opts)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#report_web_site threw exception: #{e.message}"
+      elog "Problem reporting web site: #{e.message}"
     end
   end
 end

lib/metasploit/framework/data_service/proxy/workspace_data_proxy.rb

@@ -5,7 +5,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.find_workspace(workspace_name)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#find_workspace threw exception: #{e.message}"
+      elog "Problem finding workspace: #{e.message}"
     end
   end
 
@@ -14,7 +14,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.add_workspace(workspace_name)
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#add_workspace threw exception: #{e.message}"
+      elog "Problem adding workspace: #{e.message}"
     end
   end
 
@@ -23,7 +23,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.default_workspace
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#default_workspace threw exception: #{e.message}"
+      elog "Problem getting the default workspace: #{e.message}"
     end
   end
 
@@ -32,7 +32,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.workspace
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#workspace threw exception: #{e.message}"
+      elog "Problem retrieving workspace: #{e.message}"
     end
   end
 
@@ -41,7 +41,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.workspace = workspace
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#find_workspace threw exception: #{e.message}"
+      elog "Problem setting workspace: #{e.message}"
     end
   end
 
@@ -50,7 +50,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.workspaces
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#workspaces threw exception: #{e.message}"
+      elog "Problem retrieving workspaces: #{e.message}"
     end
   end
 
@@ -59,7 +59,7 @@ module WorkspaceDataProxy
       data_service = self.get_data_service()
       data_service.workspace_associations_counts()
     rescue  Exception => e
-      puts"Call to  #{data_service.class}#workspace_associations_counts threw exception: #{e.message}"
+      elog "Problem retrieving workspaces counts: #{e.message}"
     end
   end
 

lib/metasploit/framework/data_service/remote/http/core.rb

@@ -31,6 +31,18 @@ class RemoteHTTPDataService
     build_client_pool(5)
   end
 
+  def connection_established?
+    true
+  end
+
+  def after_establish_connection
+
+  end
+
+  def error
+    'none'
+  end
+
   #
   # POST data to the HTTP endpoint and don't wait for the endpoint to process the data before getting a response
   #
@@ -111,7 +123,7 @@ class RemoteHTTPDataService
       # simplify query by removing nil values
       query_str = (!query.nil? && !query.empty?) ? append_workspace(query).compact.to_query : nil
       uri = URI::HTTP::build({path: path, query: query_str})
-      puts "#{Time.now} - HTTP #{request_type} request to #{uri.request_uri} with #{data_hash ? data_hash : "nil"}"
+      dlog("HTTP #{request_type} request to #{uri.request_uri} with #{data_hash ? data_hash : "nil"}")
 
       client = @client_pool.pop()
       case request_type
@@ -131,20 +143,17 @@ class RemoteHTTPDataService
 
       case response
         when Net::HTTPOK
-          # puts 'request sent successfully'
           return SuccessResponse.new(response)
         else
-          puts "HTTP #{request_type} request: #{uri.request_uri} failed with code: #{response.code} message: #{response.body}"
+          ilog "HTTP #{request_type} request: #{uri.request_uri} failed with code: #{response.code} message: #{response.body}"
           return FailedResponse.new(response)
       end
     rescue EOFError => e
-      puts "ERROR: No data was returned from the server."
-      puts "Backtrace: #{e.message}"
-      e.backtrace.each { |line| puts "#{line}\n"}
-      return FailedResponse.new("")
+      elog "No data was returned from the data service for request type/path : #{request_type}/#{path}, message: #{e.message}"
+      return FailedResponse.new('')
     rescue Exception => e
-      puts "Problem with HTTP #{request_type} request: #{e.message}"
-      e.backtrace.each { |line| puts "#{line}\n" }
+      elog "Problem with HTTP request for type/path: #{request_type}/#{path} message: #{e.message}"
+      return FailedResponse.new('')
     ensure
       @client_pool << client
     end
@@ -157,32 +166,6 @@ class RemoteHTTPDataService
     return true
   end
 
-  # def do_nl_search(search)
-  #   search_item = search.query.split(".")[0]
-  #   case search_item
-  #     when "host"
-  #       do_host_search(search)
-  #   end
-  # end
-
-  # def active
-  #   begin
-  #     request_opts = {'method' => 'GET', 'uri' => ONLINE_TEST_URL}
-  #     request = @client.request_raw(request_opts)
-  #     response = @client._send_recv(request)
-  #     if response.code == 200
-  #       try_sound_effect()
-  #       return true
-  #     else
-  #       puts "request failed with code: #{response.code} message: #{response.message}"
-  #       return false
-  #     end
-  #   rescue Exception => e
-  #     puts "Unable to contact goliath service: #{e.message}"
-  #     return false
-  #   end
-  # end
-
   def name
     "remote_data_service: (#{@endpoint})"
   end
@@ -254,10 +237,10 @@ class RemoteHTTPDataService
     if !data_hash.nil? && !data_hash.empty?
       data_hash.each do |k,v|
         if v.is_a?(Msf::Session)
-          puts "#{Time.now} - DEBUG: Dropping Msf::Session object before converting to JSON."
-          puts "data_hash is #{data_hash}"
-          puts "Callstack:"
-          caller.each { |line| puts "#{line}\n"}
+          dlog('Dropping Msf::Session object before converting to JSON.')
+          dlog("data_hash is #{data_hash}")
+          dlog('Callstack:')
+          caller.each { |line| dlog("#{line}\n")}
           data_hash.delete(k)
         end
       end
@@ -313,11 +296,6 @@ class RemoteHTTPDataService
     true
   end
 
-  def try_sound_effect()
-    sound_file = ::File.join(Msf::Config.data_directory, "sounds", "Goliath_Online_Sound_Effect.wav")
-    Rex::Compat.play_sound(sound_file)
-  end
-
 end
 end
 end

lib/metasploit/framework/data_service/remote/msf_red/job_handler.rb

@@ -1,9 +0,0 @@
-module JobHandler
-  def handle(job_details)
-    raise 'JobHandler#handle is not implemented';
-  end
-
-  def job_type_handled
-    raise 'JobHandler#job_type_handled is not implemented';
-  end
-end

lib/metasploit/framework/data_service/remote/msf_red/job_handlers/message_job_handler.rb

@@ -1,23 +0,0 @@
-require 'metasploit/framework/data_service/remote/msf_red/job_handler'
-
-class MessageJobHandler
-  include JobHandler
-
-  JOB_HANDLED = 'message'
-
-  def handle(message_hash)
-    message = "User: #{message_hash['user_id']}, #{message_hash['message']}"
-    banner = "*" * message.size
-    puts "\n"
-    puts "\n"
-    puts banner
-    puts message
-    puts banner
-    puts "\n"
-  end
-
-  def job_type_handled
-    JOB_HANDLED
-  end
-
-end

lib/metasploit/framework/data_service/remote/msf_red/msf_red_service.rb

@@ -1,122 +0,0 @@
-require 'metasploit/framework/data_service'
-require 'metasploit/framework/data_service/remote/http/core'
-
-class MSFRedService
-  JOB_CHECK_INTERVAL_SEC = 5
-  LOGIN_TIMEOUT_SEC = 10
-  SESSION_KEY_VALUE = 'msf-session-key'
-  LOGIN_ENDPOINT = '/login'
-  JOBS_ENDPOINT = '/jobs'
-  CONSOLE_SERVICE_HOST_NAME = 'console-service.metasploit.r7ops.com'
-  CONSOLE_SERVICE_PORT = 8080
-
-  def initialize
-    @client = Rex::Proto::Http::Client.new(CONSOLE_SERVICE_HOST_NAME, CONSOLE_SERVICE_PORT)
-    @job_handlers = Hash.new()
-    load_job_handlers
-  end
-
-  # TODO: Obviously this is not secure
-  def launch(username, password)
-    if (do_login(username, password))
-      inject_data_service
-      start_job_thread
-    end
-
-  end
-
-  #######
-  private
-  #######
-
-  def load_job_handlers
-      job_handler_path = File.dirname(__FILE__) + '/job_handlers/*'
-      Dir.glob(job_handler_path).collect{|file_path|
-        job_handler_class = File.basename(file_path, '.rb').classify
-        require file_path
-        job_handler_class_constant = job_handler_class.constantize
-        job_handler = job_handler_class_constant.new
-        @job_handlers[job_handler.job_type_handled] = job_handler
-      }
-  end
-
-  def inject_data_service
-    endpoint = URI.parse("http://#{CONSOLE_SERVICE_HOST_NAME}:#{CONSOLE_SERVICE_PORT}")
-    remote_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint)
-    remote_data_service.set_header(SESSION_KEY_VALUE, @session_key)
-    data_service_manager = Metasploit::Framework::DataService::DataProxy.instance
-    data_service_manager.register_data_service(remote_data_service)
-  end
-
-  def do_login(username, password)
-    login_hash = {:username => username, :password => password}
-    begin
-
-      request_opts = { 'method' => 'POST', 'ctype' => 'application/json', 'uri' => LOGIN_ENDPOINT, 'data' => login_hash.to_json }
-      request = @client.request_raw(request_opts)
-      response = @client._send_recv(request, LOGIN_TIMEOUT_SEC)
-
-      if response.code == 200
-        data = JSON.parse(response.body)
-        @session_key =  data['session_key']
-        puts "MSF Red console login successfull, session: #{@session_key}"
-        return true
-      else
-        puts "Login failed: failed with code: #{response.code} message: #{response.body}"
-        return false
-      end
-    rescue Exception => e
-      puts "Problem with POST request: #{e.message}"
-      return false
-    end
-  end
-
-
-  def start_job_thread
-    Thread.start {
-      loop  {
-        sleep 5
-        begin
-          job_hash = get_next_job
-          if (job_hash.nil? or job_hash.empty?)
-            next
-          end
-
-          type = job_hash['job_type']
-          job_handler = @job_handlers[type]
-          if (job_handler.nil?)
-            puts "No registered job handler for type: #{type}"
-          else
-            job_handler.handle(job_hash['job_details'])
-          end
-        rescue Exception => e
-          puts "Problem executing job: #{e.message}"
-        end
-      }
-    }
-  end
-
-  def get_next_job
-    request_opts = { 'method' => 'GET', 'ctype' => 'application/json', 'uri' => JOBS_ENDPOINT, 'headers' => {SESSION_KEY_VALUE => @session_key} }
-    request = @client.request_raw(request_opts)
-    response = @client._send_recv(request)
-
-    if response.code == 200
-      if (response.body.nil? or response.body.empty?)
-        return nil
-      end
-
-      begin
-        return JSON.parse(response.body)
-      rescue Exception => e
-        puts "Unable to parse: #{response.body}, reason: #{e.message}"
-        return nil
-      end
-
-    else
-      puts "GET request: #{path} with body: #{json_body} failed with code: #{response.code} message: #{response.body}"
-      return nil
-    end
-  end
-
-end

lib/metasploit/framework/parsed_options/base.rb

@@ -137,10 +137,6 @@ class Metasploit::Framework::ParsedOptions::Base
         options.database.config = path
       end
 
-      option_parser.on('-dbrp', 'Run database as a separate local process') do
-        options.database.remote_process = true
-      end
-
       option_parser.separator ''
       option_parser.separator 'Framework options'
 

lib/metasploit/framework/parsed_options/remote_db.rb

@@ -22,7 +22,9 @@ class Metasploit::Framework::ParsedOptions::RemoteDB < Metasploit::Framework::Pa
     @options
   end
 
+  #######
   private
+  #######
 
   def option_parser
     unless @option_parser

lib/msf/core/db_manager/http/aws/sns_handler.rb

@@ -1,48 +0,0 @@
-require 'net/http'
-require 'msf/core/db_manager/http/servlet_helper'
-
-class SNSHandler
-  include ServletHelper
-
-  def initialize(app)
-    @app = app
-  end
-
-  def call(env)
-    request = Rack::Request.new(env)
-    # puts "Received #{env['REQUEST_METHOD']} for path #{env['REQUEST_PATH']}"
-    sns_message_type = env['HTTP_X_AMZ_SNS_MESSAGE_TYPE']
-    if (request.post? and not sns_message_type.nil?)
-      case sns_message_type
-        when "Notification"
-          env['rack.input'] = get_message_io(request)
-        when "SubscriptionConfirmation"
-          do_confirm(request)
-          return [200, {}, ['']]
-      end
-    end
-
-    @app.call(env)
-  end
-
-  #######
-  private
-  #######
-
-  # Confirms SNS subscription
-  def do_confirm(request)
-    opts = parse_json_request(request, true)
-    subscription_url = opts[:SubscribeURL]
-    begin
-      Net::HTTP.get(URI(subscription_url))
-    rescue Exception => e
-      puts "Error on subscription: #{e.message}"
-    end
-  end
-
-  def get_message_io(request)
-    opts = parse_json_request(request, true)
-    message = opts[:Message]
-    return StringIO.new(message)
-  end
-end

lib/msf/core/db_manager/http/aws/sns_message.rb

@@ -1,2 +0,0 @@
-class SNSMessage
-end

lib/msf/core/db_manager/http/http_db_manager_service.rb

@@ -1,19 +1,12 @@
 require 'rack'
 require 'msf/core/db_manager/http/sinatra_app'
 require 'metasploit/framework/parsed_options/remote_db'
+require 'rex/ui/text/output/stdio'
 
 class HttpDBManagerService
 
   def start(opts)
     parsed_options = Metasploit::Framework::ParsedOptions::RemoteDB.new
-    if (parsed_options.options.database.no_signal)
-      puts 'removing trap'
-      opts[:signals] = false
-      @shutdown_on_interupt = false
-    else
-      @shutdown_on_interupt = true
-    end
-
     require_environment!(parsed_options)
 
     if opts[:ssl]
@@ -35,17 +28,12 @@ class HttpDBManagerService
 
     Rack::Handler::Thin.run(SinatraApp, opts) do |server|
 
-      # TODO: prevent accidental shutdown from msfconle eg: ctrl-c
-      [:INT, :TERM].each { |sig|
-        trap(sig) {
-          server.stop if (@shutdown_on_interupt || sig == :TERM)
-        }
-      }
-
       if opts[:ssl] && opts[:ssl] = true
-        puts "Starting in HTTPS mode"
+        print_good "SSL Enabled"
         server.ssl = true
         server.ssl_options = opts[:ssl_opts]
+      else
+        print_warning 'SSL Disabled'
       end
       server.threaded = true
     end
@@ -84,4 +72,33 @@ class HttpDBManagerService
   #   }
   # end
 
-end
+
+
+end
+
+
+def print_line(msg)
+  $console_printer.print_line(msg)
+end
+
+def print_warning(msg)
+  $console_printer.print_warning(msg)
+end
+
+def print_good(msg)
+  $console_printer.print_good(msg)
+end
+
+def print_error(msg, exception = nil)
+  unless exception.nil?
+    msg += "\n    Call Stack:"
+    exception.backtrace.each {|line|
+      msg += "\n"
+      msg += "\t #{line}"
+    }
+  end
+
+  $console_printer.print_error(msg)
+end
+
+$console_printer = Rex::Ui::Text::Output::Stdio.new

lib/msf/core/db_manager/http/job_processor.rb

@@ -21,8 +21,7 @@ class JobProcessor
         begin
           wrapper.job.call(wrapper.job_args)
         rescue Exception => e
-          puts "Error executing job #{e.message}"
-          e.backtrace.each { |line| puts "#{line}\n"}
+          print_error "Error executing job #{e.message}", e
         end
       end
     }

lib/msf/core/db_manager/http/servlet_helper.rb

@@ -7,8 +7,7 @@ module ServletHelper
   include ResponseDataHelper
 
   def set_error_on_response(error)
-    puts "Error handling request: #{error.message}"
-    error.backtrace.each { |line| puts "#{line}\n" }
+    print_error "Error handling request: #{error.message}", error
     headers = {'Content-Type' => 'text/plain'}
     [500, headers, error.message]
   end

lib/msf/core/db_manager/http/sinatra_app.rb

@@ -1,6 +1,5 @@
 require 'sinatra/base'
 require 'msf/core/db_manager/http/servlet_helper'
-require 'msf/core/db_manager/http/aws/sns_handler'
 require 'msf/core/db_manager/http/servlet/host_servlet'
 require 'msf/core/db_manager/http/servlet/note_servlet'
 require 'msf/core/db_manager/http/servlet/vuln_servlet'
@@ -18,8 +17,6 @@ require 'msf/core/db_manager/http/servlet/nmap_servlet'
 
 class SinatraApp < Sinatra::Base
 
-  use SNSHandler
-
   helpers ServletHelper
 
   # Servlet registration

lib/msf/core/framework.rb

@@ -272,7 +272,7 @@ protected
   private
 
   def get_db
-    if !options['DatabaseRemoteProcess'] && !options['DisableDatabase']
+    if !options['DisableDatabase']
       db_manager = Msf::DBManager.new(self)
       db_manager.init_db(options)
       options[:db_manager] = db_manager

lib/msf/ui/console/command_dispatcher/core.rb

@@ -20,7 +20,6 @@ require 'msf/ui/console/command_dispatcher/jobs'
 require 'msf/ui/console/command_dispatcher/resource'
 require 'msf/ui/console/command_dispatcher/modules'
 require 'msf/util/document_generator'
-require 'metasploit/framework/data_service/remote/msf_red/msf_red_service'
 
 module Msf
 module Ui
@@ -108,7 +107,6 @@ class Core
       "?"          => "Help menu",
       "banner"     => "Display an awesome metasploit banner",
       "cd"         => "Change the current working directory",
-      "msf_red_connect" => "Connect to MSF Platform",
       "connect"    => "Communicate with a host",
       "color"      => "Toggle color",
       "exit"       => "Exit the console",
@@ -263,20 +261,6 @@ class Core
 
   end
 
-  def cmd_msf_red_connect(*args)
-    while (arg = args.shift)
-      case arg
-        when '-u'
-          username = args.shift
-        when '-p'
-          password = args.shift
-      end
-    end
-
-    msf_red_service = MSFRedService.new()
-    msf_red_service.launch(username, password)
-  end
-
   def cmd_connect_help
     print_line "Usage: connect [options] <host> <port>"
     print_line
@@ -473,7 +457,6 @@ class Core
     forced = false
     forced = true if (args[0] and args[0] =~ /-y/i)
 
-    framework.db.exit_called
     if(framework.sessions.length > 0 and not forced)
       print_status("You have active sessions open, to exit anyway type \"exit -y\"")
       return

lib/msf/ui/console/command_dispatcher/db.rb

@@ -47,13 +47,7 @@ module Msf
                 "db_export"     => "Export a file containing the contents of the database",
                 "db_nmap"       => "Executes nmap and records the output automatically",
                 "db_rebuild_cache" => "Rebuilds the database-stored module cache",
-                "test_data_service_host" => "Blah",
-                "test_data_service_loot" => "Blah",
-                "perf_test_data_service_loot" => "REMOVEME - Run a performance test against the loot data service",
-                "add_data_service" => "Blah",
-                "list_data_services" => "Blah",
-                "set_data_service" => "Blah",
-                "nl_search" => "Blah"
+                "data_services" => "Command to add, list and set a data service",
             }
 
             # Always include commands that only make sense when connected.
@@ -88,147 +82,24 @@ module Msf
             true
           end
 
-          def cmd_set_data_service(service_id)
-            framework.db.set_data_service(service_id)
-          end
-
-          def cmd_list_data_services()
-            framework.db.print_data_services
-          end
-
-          def cmd_add_data_service(*args)
-            protocol = "http"
-            port = 80
-            https_opts = {}
+          def cmd_data_services(*args)
             while (arg = args.shift)
               case arg
                 when '-h', '--help'
-                  cmd_add_data_service_help
+                  data_service_help
+                  return
+                when '-a', '--add'
+                  add_data_service(*args)
+                  return
+                when '-s', '--set'
+                  set_data_service(args.shift)
                   return
-                when '-p'
-                  port = args.shift
-                when '-s', '--ssl'
-                  protocol = "https"
-                when '-c', '--cert'
-                  https_opts[:cert] = args.shift
-                when '--skip-verify'
-                  https_opts[:skip_verify] = true
-                else
-                  host = arg
               end
             end
 
-            if host.nil? || port.nil?
-              print_error "Host and port are required."
-              return
-            end
-
-            endpoint = "#{protocol}://#{host}:#{port}"
-            remote_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint, https_opts)
-            framework.db.register_data_service(remote_data_service)
+            list_data_services
           end
 
-          def cmd_add_data_service_help
-            print_line "Usage: add_data_service [ options ] [ Remote Address]"
-            print_line
-            print_line "OPTIONS:"
-            print_line "  -h, --help        Show this help information."
-            print_line "  -p <port>         The port the data service is listening on. Default is 80."
-            print_line "  -s, --ssl         Enable SSL. Required for HTTPS data services."
-            print_line "  -c, --cert        Certificate file matching the server's certificate. Needed when using self-signed SSL cert."
-            print_line "  --skip-verify     Skip validating authenticity of server's certificate. NOT RECOMMENDED."
-            print_line
-          end
-
-          def cmd_test_data_service_host(*args)
-            host = {}
-            while (arg = args.shift)
-              case arg
-                when '-h'
-                  host[:host] = args.shift
-                when '-n'
-                  host[:name] = args.shift
-                when '-o'
-                  host[:os_name] = args.shift
-              end
-            end
-
-            puts 'Reporting test host to data service'
-            framework.db.report_host host
-          end
-
-          def cmd_test_data_service_loot(*args)
-            loot = {}
-            while (arg = args.shift)
-              case arg
-                when '-h'
-                  loot[:host] = args.shift
-                when '-n'
-                  loot[:name] = args.shift
-                when '-s'
-                  loot[:service] = args.shift
-                when '-t'
-                  loot[:type] = args.shift
-                when '-i'
-                  loot[:info] = args.shift
-                when '-p'
-                  loot[:path] = args.shift
-              end
-            end
-
-            puts 'Reporting test loot to data service'
-            framework.db.report_loot loot
-          end
-
-          def cmd_perf_test_data_service_loot(*args)
-            host = "172.28.128.3"
-            name = SecureRandom.uuid
-            type = "file"
-            info = "file"
-            path = "/Users/jbarnett/.msf4/loot/20171005151912_default_172.28.128.3_linux.passwd_256904.txt"
-            data = File.read('/Users/jbarnett/rapid7/goliath/base64.txt')
-
-            while (arg = args.shift)
-              case arg
-                when '-h'
-                  host = args.shift
-                when '-n'
-                  name = args.shift
-                when '-s'
-                  service = args.shift
-                when '-t'
-                  type = args.shift
-                when '-i'
-                  info = args.shift
-                when '-p'
-                  path = args.shift
-                when '-d'
-                  data = args.shift
-                when '-c'
-                  count = args.shift.to_i
-              end
-            end
-
-            loots = []
-            count.times do
-              loots << { :host => host,
-                         :name => name,
-                         :type => type,
-                         :info => info,
-                         :path => path,
-                         :data => data
-              }
-            end
-
-            puts 'Reporting test loot to data service'
-            start_time = Time.now
-            puts "#{start_time} - Staring loot perf test"
-            loots.each do |loot|
-              framework.db.report_loot loot
-            end
-            end_time = Time.now
-            puts "#{end_time} - Ending loot perf test. Duration was #{end_time - start_time}"
-          end
 
           def cmd_workspace_help
             print_line "Usage:"
@@ -2006,7 +1877,77 @@ module Msf
             end
           end
 
+          #######
           private
+          #######
+
+          def add_data_service(*args)
+            protocol = "http"
+            port = 8080
+            https_opts = {}
+            while (arg = args.shift)
+              case arg
+                when '-p'
+                  port = args.shift
+                when '-s', '--ssl'
+                  protocol = "https"
+                when '-c', '--cert'
+                  https_opts[:cert] = args.shift
+                when '--skip-verify'
+                  https_opts[:skip_verify] = true
+                else
+                  host = arg
+              end
+            end
+
+            if host.nil? || port.nil?
+              print_error "Host and port are required."
+              return
+            end
+
+            endpoint = "#{protocol}://#{host}:#{port}"
+            remote_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint, https_opts)
+            begin
+              framework.db.register_data_service(remote_data_service)
+              print_line "Registered data service: #{remote_data_service.name}"
+            rescue Exception => e
+              print_error "There was a problem registering the remote data service: #{e.message}"
+            end
+          end
+
+          def set_data_service(service_id)
+            begin
+              framework.db.set_data_service(service_id)
+            rescue Exception => e
+              print_error "Unable to set data service: #{e.message}"
+            end
+          end
+
+          def list_data_services()
+            framework.db.get_services_metadata.each {|metadata|
+              out = "id: #{metadata.id}, name: #{metadata.name}"
+              if metadata.active
+                out += " [active]"
+              end
+              print_line out
+            }
+          end
+
+          def data_service_help
+            print_line "Usage: data_services [ options ] - list data services by default"
+            print_line
+            print_line "OPTIONS:"
+
+            print_line "  -h, --help                  Show this help information."
+            print_line "  -s, --set <id>              Set the data service by identifier."
+            print_line "  -a, --add [ options ] host  Adds data service"
+            print_line "  Add Data Service Options:"
+            print_line "  -p <port>         The port the data service is listening on. Default is 8080."
+            print_line "  -s, --ssl         Enable SSL. Required for HTTPS data services."
+            print_line "  -c, --cert        Certificate file matching the server's certificate. Needed when using self-signed SSL cert."
+            print_line "  --skip-verify     Skip validating authenticity of server's certificate. NOT RECOMMENDED."
+            print_line
+          end
 
           def print_msgs(status_msg, error_msg)
             status_msg.each do |s|

msfdb_ws

@@ -20,7 +20,7 @@ end
 def parse_args(args)
   opts = {}
   opt = OptionParser.new
-  banner = "msfdb - A remote database process for Metasploit Framework.\n"
+  banner = "msfdb_ws - Metasploit data store as a web service.\n"
   banner << "Usage: #{$0} [options] <var=val>"
   opt.banner = banner
   opt.separator('')
@@ -70,7 +70,7 @@ end
 
 begin
   opts = parse_args(ARGV)
-  raise SwitchError.new("certificate file and key file must be specified when using -s") if opts[:ssl] && (opts[:ssl_key].nil? || opts[:ssl_cert].nil?)
+  raise SwitchError.new("certificate file must be specified when using -s") if opts[:ssl] && (opts[:ssl_cert].nil?)
   HttpDBManagerService.new.start(:Port => opts[:port],
                                  :Host => opts[:interface],
                                  :ssl => opts[:ssl],