Land #10024, Fix find_or_create_* methods for remote data service

This PR updates the find_or_create_* methods associated with each model to
no longer just proxy to the report_* model. It now performs a lookup through
the DataProxy and returns the found object if it exists, or creates a new
record if needed.
GSoC/Meterpreter_Web_Console
James Barnett 2018-05-22 17:08:46 -05:00
commit 0472b9df3f
No known key found for this signature in database
GPG Key ID: 647983861A4EC5EA
26 changed files with 292 additions and 153 deletions

View File

@ -1,13 +1,12 @@
module HostDataProxy
def hosts(wspace = workspace.name, non_dead = false, addresses = nil, search_term = nil)
def hosts(opts = {})
begin
data_service = self.get_data_service
opts = {}
add_opts_workspace(opts, wspace)
opts[:non_dead] = non_dead
opts[:address] = addresses
opts[:search_term] = search_term
opts[:non_dead] = false unless opts.has_key?(:non_dead)
opts[:address] = opts.delete(:address) || opts.delete(:host)
opts[:search_term] = nil unless opts.has_key?(:search_term)
add_opts_workspace(opts)
data_service.hosts(opts)
rescue => e
self.log_error(e, "Problem retrieving hosts")
@ -15,17 +14,24 @@ module HostDataProxy
end
def find_or_create_host(opts)
host = get_host(opts)
return host unless host.nil?
report_host(opts)
begin
host = hosts(opts.clone)
if host.nil? || host.first.nil?
host = report_host(opts.clone)
else
host = host.first
end
host
rescue => e
self.log_error(e, "Problem finding or creating host")
end
end
def get_host(opts)
begin
data_service = self.get_data_service()
data_service.get_host(opts)
rescue e
rescue => e
self.log_error(e, "Problem retrieving host")
end
end

View File

@ -13,16 +13,32 @@ module LootDataProxy
end
end
# TODO: Shouldn't this proxy to RemoteLootDataService#find_or_create_loot ?
# It's currently skipping the "find" part
def find_or_create_loot(opts)
report_loot(opts)
begin
# create separate opts for find operation since the report operation uses slightly different keys
# TODO: standardize option keys used for the find and report operations
find_opts = opts.clone
# convert type to ltype
find_opts[:ltype] = find_opts.delete(:type) if find_opts.key?(:type)
# convert host to nested hosts address
find_opts[:hosts] = {address: find_opts.delete(:host)} if find_opts.key?(:host)
loot = loots(find_opts)
if loot.nil? || loot.first.nil?
loot = report_loot(opts.clone)
else
loot = loot.first
end
loot
rescue => e
self.log_error(e, "Problem finding or creating loot")
end
end
def loots(wspace, opts = {})
def loots(opts = {})
begin
data_service = self.get_data_service
add_opts_workspace(opts, wspace)
add_opts_workspace(opts)
data_service.loot(opts)
rescue => e
self.log_error(e, "Problem retrieving loot")

View File

@ -10,9 +10,26 @@ module NoteDataProxy
end
end
# TODO: like other *DataProxy modules this currently skips the "find" part
def find_or_create_note(opts)
report_note(opts)
begin
# create separate opts for find operation since the report operation uses slightly different keys
# TODO: standardize option keys used for the find and report operations
find_opts = opts.clone
# convert type to ntype
find_opts[:ntype] = find_opts.delete(:type) if find_opts.key?(:type)
# convert host to nested hosts address
find_opts[:hosts] = {address: find_opts.delete(:host)} if find_opts.key?(:host)
note = notes(find_opts)
if note.nil? || note.first.nil?
note = report_note(opts.clone)
else
note = note.first
end
note
rescue => e
self.log_error(e, "Problem finding or creating note")
end
end
def report_note(opts)

View File

@ -1,9 +1,9 @@
module ServiceDataProxy
def services(wspace = workspace.name, opts = {})
def services(opts = {})
begin
data_service = self.get_data_service
add_opts_workspace(opts, wspace)
add_opts_workspace(opts)
data_service.services(opts)
rescue => e
self.log_error(e, 'Problem retrieving services')
@ -11,7 +11,23 @@ module ServiceDataProxy
end
def find_or_create_service(opts)
report_service(opts)
begin
# create separate opts for find operation since the report operation uses slightly different keys
# TODO: standardize option keys used for the find and report operations
find_opts = opts.clone
# convert host to nested hosts address
find_opts[:hosts] = {address: find_opts.delete(:host)} if find_opts.key?(:host)
service = services(find_opts)
if service.nil? || service.first.nil?
service = report_service(opts.clone)
else
service = service.first
end
service
rescue => e
self.log_error(e, "Problem finding or creating service")
end
end
def report_service(opts)

View File

@ -11,6 +11,20 @@ module VulnDataProxy
end
end
def find_or_create_vuln(opts)
begin
vuln = vulns(opts.clone)
if vuln.nil? || vuln.first.nil?
vuln = report_vuln(opts.clone)
else
vuln = vuln.first
end
vuln
rescue => e
self.log_error(e, "Problem finding or creating vuln")
end
end
def report_vuln(opts)
begin
data_service = self.get_data_service

View File

@ -19,10 +19,6 @@ module RemoteHostDataService
json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
end
def find_or_create_host(opts)
json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
end
def report_hosts(hosts)
self.post_data(HOST_API_PATH, hosts)
end

View File

@ -23,10 +23,6 @@ module RemoteLootDataService
self.post_data_async(LOOT_API_PATH, opts)
end
def find_or_create_loot(opts)
json_to_mdm_object(self.post_data(LOOT_API_PATH, opts), LOOT_MDM_CLASS, [])
end
def report_loots(loot)
self.post_data(LOOT_API_PATH, loot)
end

View File

@ -20,6 +20,10 @@ module HostDataService
raise 'HostDataService#find_or_create_host is not implemented'
end
def update_host(opts)
raise 'HostDataService#update_host is not implemented'
end
def delete_host(opts)
raise 'HostDataService#delete_host is not implemented'
end

View File

@ -4,7 +4,15 @@ module LootDataService
raise 'LootDataService#report_loot is not implemented'
end
def find_or_create_loot(opts)
raise 'LootDataService#find_or_create_loot is not implemented'
end
def loot(opts)
raise 'LootDataService#loots is not implemented'
end
def update_loot(opts)
raise 'LootDataService#update_loot is not implemented'
end
end

View File

@ -4,6 +4,10 @@ module NoteDataService
raise 'NoteDataService#notes is not implemented'
end
def find_or_create_note(opts)
raise 'NoteDataService#find_or_create_note is not implemented'
end
def report_note(opts)
raise 'NoteDataService#report_note is not implemented'
end
@ -15,5 +19,4 @@ module NoteDataService
def delete_note(opts)
raise 'NoteDataService#delete_note is not implemented'
end
end

View File

@ -1,7 +1,22 @@
module ServiceDataService
def services(opts)
raise 'ServiceDataService#services is not implemented'
end
def find_or_create_service(opts)
raise 'ServiceDataService#find_or_create_service is not implemented'
end
def report_service(opts)
raise 'ServiceDataService#report_service is not implemented'
end
def update_service(opts)
raise 'ServiceDataService#update_service is not implemented'
end
def delete_service(opts)
raise 'ServiceDataService#delete_service is not implemented'
end
end

View File

@ -1,7 +1,22 @@
module VulnDataService
def report_vuln(opts)
raise 'VulnDataServicee#report_vuln is not implemented'
def vulns(opts)
raise 'VulnDataService#vulns is not implemented'
end
def find_or_create_vuln(opts)
raise 'VulnDataService#find_or_create_vuln is not implemented'
end
def report_vuln(opts)
raise 'VulnDataService#report_vuln is not implemented'
end
def update_vuln(opts)
raise 'VulnDataService#update_vuln is not implemented'
end
def delete_vuln(opts)
raise 'VulnDataService#delete_vuln is not implemented'
end
end

View File

@ -20,11 +20,15 @@ module WorkspaceDataService
raise 'WorkspaceDataService#workspace= is not implemented'
end
def workspaces
def workspaces(opts)
raise 'WorkspaceDataService#workspaces is not implemented'
end
def workspace_associations_counts()
raise 'WorkspaceDataService#workspace_associations_counts is not implemented'
def delete_workspaces(opts)
raise 'WorkspaceDataService#delete_workspaces is not implemented'
end
def update_workspace(opts)
raise 'WorkspaceDataService#update_workspace is not implemented'
end
end

View File

@ -12,8 +12,8 @@ module Msf::DBManager::Event
return if not wspace # Temp fix?
uname = opts.delete(:username)
if ! opts[:host].kind_of? ::Mdm::Host and opts[:host]
opts[:host] = report_host(:workspace => wspace, :host => opts[:host])
if !opts[:host].nil? && !opts[:host].kind_of?(::Mdm::Host)
opts[:host] = find_or_create_host(workspace: wspace, host: opts[:host])
end
::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname))

View File

@ -58,7 +58,7 @@ module Msf::DBManager::Host
ip = opts[:ip]
tag_name = opts[:tag_name]
host = framework.db.get_host(:workspace => wspace, :address => ip)
host = get_host(workspace: wspace, address: ip)
if host
possible_tags = Mdm::Tag.joins(:hosts).where("hosts.workspace_id = ? and hosts.address = ? and tags.name = ?", wspace.id, ip, tag_name).order("tags.id DESC").limit(1)
tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first)
@ -185,74 +185,78 @@ module Msf::DBManager::Host
::ActiveRecord::Base.connection_pool.with_connection {
wspace = Msf::Util::DBManager.process_opts_workspace(opts, framework)
ret = { }
if !addr.kind_of? ::Mdm::Host
addr = Msf::Util::Host.normalize_host(addr)
unless ipv46_validator(addr)
raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
end
if opts[:comm] and opts[:comm].length > 0
host = wspace.hosts.where(address: addr, comm: opts[:comm]).first_or_initialize
else
host = wspace.hosts.where(address: addr).first_or_initialize
end
else
host = addr
end
ostate = host.state
# Truncate the info field at the maximum field length
if opts[:info]
opts[:info] = opts[:info][0,65535]
end
# Truncate the name field at the maximum field length
if opts[:name]
opts[:name] = opts[:name][0,255]
end
if opts[:os_name]
os_name, os_flavor = split_windows_os_name(opts[:os_name])
opts[:os_name] = os_name if os_name.present?
if opts[:os_flavor].present?
opts[:os_flavor] = os_flavor + opts[:os_flavor]
else
opts[:os_flavor] = os_flavor
end
end
opts.each do |k,v|
if (host.attribute_names.include?(k.to_s))
unless host.attribute_locked?(k.to_s)
host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
end
elsif !v.blank?
dlog("Unknown attribute for ::Mdm::Host: #{k}")
end
end
host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info
# Set default fields if needed
host.state = Msf::HostState::Alive if !host.state
host.comm = '' if !host.comm
host.workspace = wspace if !host.workspace
begin
framework.events.on_db_host(host) if host.new_record?
rescue ::Exception => e
wlog("Exception in on_db_host event handler: #{e.class}: #{e}")
wlog("Call Stack\n#{e.backtrace.join("\n")}")
end
retry_attempts ||= 0
if !addr.kind_of? ::Mdm::Host
addr = Msf::Util::Host.normalize_host(addr)
host_state_changed(host, ostate) if host.state != ostate
unless ipv46_validator(addr)
raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
end
if host.changed?
msf_import_timestamps(opts,host)
host.save!
conditions = {address: addr}
conditions[:comm] = opts[:comm] if !opts[:comm].nil? && opts[:comm].length > 0
host = wspace.hosts.where(conditions).first_or_initialize
else
host = addr
end
ostate = host.state
# Truncate the info field at the maximum field length
if opts[:info]
opts[:info] = opts[:info][0,65535]
end
# Truncate the name field at the maximum field length
if opts[:name]
opts[:name] = opts[:name][0,255]
end
if opts[:os_name]
os_name, os_flavor = split_windows_os_name(opts[:os_name])
opts[:os_name] = os_name if os_name.present?
if opts[:os_flavor].present?
opts[:os_flavor] = os_flavor + opts[:os_flavor]
else
opts[:os_flavor] = os_flavor
end
end
opts.each do |k,v|
if host.attribute_names.include?(k.to_s)
unless host.attribute_locked?(k.to_s)
host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
end
elsif !v.blank?
dlog("Unknown attribute for ::Mdm::Host: #{k}")
end
end
host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info
# Set default fields if needed
host.state = Msf::HostState::Alive unless host.state
host.comm = '' unless host.comm
host.workspace = wspace unless host.workspace
begin
framework.events.on_db_host(host) if host.new_record?
rescue => e
wlog("Exception in on_db_host event handler: #{e.class}: #{e}")
wlog("Call Stack\n#{e.backtrace.join("\n")}")
end
host_state_changed(host, ostate) if host.state != ostate
if host.changed?
msf_import_timestamps(opts, host)
host.save!
end
rescue ActiveRecord::RecordNotUnique
# two concurrent report requests for a new host could result in a RecordNotUnique exception
# simply retry the report once more as an optimistic approach
retry if (retry_attempts+=1) <= 1
raise
end
if opts[:task]

View File

@ -4,8 +4,8 @@ module SessionServlet
end
def self.registered(app)
app.post SessionServlet.api_path, &report_session
app.get SessionServlet.api_path, &get_session
app.post SessionServlet.api_path, &report_session
end
#######
@ -16,7 +16,7 @@ module SessionServlet
lambda {
begin
#opts = parse_json_request(request, false)
data = get_db().get_all_sessions()
data = get_db.get_all_sessions()
set_json_response(data)
rescue => e
set_error_on_response(e)
@ -26,14 +26,18 @@ module SessionServlet
def self.report_session
lambda {
job = lambda { |opts|
if (opts[:session_data])
get_db().report_session_dto(opts)
else
get_db().report_session_host_dto(opts)
end
}
exec_report_job(request, &job)
begin
job = lambda { |opts|
if opts[:session_data]
get_db.report_session_dto(opts)
else
get_db.report_session_host_dto(opts)
end
}
exec_report_job(request, &job)
rescue => e
set_error_on_response(e)
end
}
end
end

View File

@ -10,6 +10,10 @@ module Msf::DBManager::Loot
# This methods returns a list of all loot in the database
#
def loots(opts)
data = opts.delete(:data)
# Remove path from search conditions as this won't accommodate remote data
# service usage where the client and server storage locations differ.
opts.delete(:path)
search_term = opts.delete(:search_term)
::ActiveRecord::Base.connection_pool.with_connection {
@ -18,10 +22,17 @@ module Msf::DBManager::Loot
if search_term && !search_term.empty?
column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Loot, search_term)
Mdm::Loot.includes(:host).where(opts).where(column_search_conditions)
results = Mdm::Loot.includes(:host).where(opts).where(column_search_conditions)
else
Mdm::Loot.includes(:host).where(opts)
results = Mdm::Loot.includes(:host).where(opts)
end
# Compare the deserialized data from the DB to the search data since the column is serialized.
unless data.nil?
results = results.select { |loot| loot.data == data }
end
results
}
end
alias_method :loot, :loots

View File

@ -25,8 +25,15 @@ module Msf::DBManager::Note
::ActiveRecord::Base.connection_pool.with_connection {
wspace = Msf::Util::DBManager.process_opts_workspace(opts, framework)
data = opts.delete(:data)
search_term = opts.delete(:search_term)
results = wspace.notes.includes(:host).where(opts)
# Compare the deserialized data from the DB to the search data since the column is serialized.
unless data.nil?
results = results.select { |note| note.data == data }
end
if search_term && !search_term.empty?
re_search_term = /#{search_term}/mi
results = results.select { |note|

View File

@ -144,15 +144,16 @@ module Msf::DBManager::Service
wspace = Msf::Util::DBManager.process_opts_workspace(opts, framework)
search_term = opts.delete(:search_term)
opts["hosts.address"] = opts.delete(:addresses)
opts.compact!
order_args = [:port]
order_args.unshift(Mdm::Host.arel_table[:address]) if opts.key?(:hosts)
::ActiveRecord::Base.connection_pool.with_connection {
if search_term && !search_term.empty?
column_search_conditions = Msf::Util::DBManager.create_all_column_search_conditions(Mdm::Service, search_term)
wspace.services.includes(:host).where(opts).where(column_search_conditions).order("hosts.address, port")
wspace.services.includes(:host).where(opts).where(column_search_conditions).order(*order_args)
else
wspace.services.includes(:host).where(opts).order("hosts.address, port")
wspace.services.includes(:host).where(opts).order(*order_args)
end
}
end

View File

@ -120,11 +120,12 @@ module Msf::DBManager::Session
::ActiveRecord::Base.connection_pool.with_connection {
host_data = session_dto[:host_data]
workspace = workspaces({ name: host_data[:workspace] })
h_opts = {}
h_opts[:host] = host_data[:host]
h_opts[:arch] = host_data[:arch]
h_opts[:workspace] = host_data[:workspace]
workspace = workspaces({ name: host_data[:workspace] }).first
h_opts = {
host: host_data[:host],
workspace: workspace
}
h_opts[:arch] = host_data[:arch] if !host_data[:arch].nil? && !host_data[:arch].empty?
host = find_or_create_host(h_opts)
session_data = session_dto[:session_data]
@ -218,7 +219,7 @@ module Msf::DBManager::Session
vuln_info[:service] = service if service
vuln = framework.db.report_vuln(vuln_info)
vuln = report_vuln(vuln_info)
attempt_info = {
host: host,
@ -233,7 +234,7 @@ module Msf::DBManager::Session
run_id: session.exploit.user_data.try(:[], :run_id)
}
framework.db.report_exploit_success(attempt_info)
report_exploit_success(attempt_info)
vuln
}
@ -245,10 +246,12 @@ module Msf::DBManager::Session
raise ArgumentError.new("Invalid :session, expected Msf::Session") unless session.kind_of? Msf::Session
wspace = opts[:workspace] || find_workspace(session.workspace)
h_opts = { }
h_opts[:host] = Msf::Util::Host.normalize_host(session)
h_opts[:arch] = session.arch if session.respond_to?(:arch) and session.arch
h_opts[:workspace] = wspace
h_opts = {
host: Msf::Util::Host.normalize_host(session),
workspace: wspace
}
h_opts[:arch] = session.arch if session.respond_to?(:arch) && !session.arch.nil? && !session.arch.empty?
host = find_or_create_host(h_opts)
sess_data = {
datastore: session.exploit_datastore.to_h,
@ -332,7 +335,7 @@ module Msf::DBManager::Session
vuln_info[:service] = service if service
vuln = framework.db.report_vuln(vuln_info)
vuln = report_vuln(vuln_info)
attempt_info = {
host: host,
@ -347,7 +350,7 @@ module Msf::DBManager::Session
run_id: vuln_info_dto[:run_id]
}
framework.db.report_exploit_success(attempt_info)
report_exploit_success(attempt_info)
vuln
}

View File

@ -251,12 +251,12 @@ class Db
tbl << [
current_workspace.name == ws.name ? '*' : '',
ws.name,
framework.db.hosts(ws.name).count,
framework.db.services(ws.name).count,
framework.db.vulns({workspace: ws.name}).count,
framework.db.creds({workspace: ws.name}).count,
framework.db.loots(ws.name).count,
framework.db.notes({workspace: ws.name}).count
framework.db.hosts(workspace: ws.name).count,
framework.db.services(workspace: ws.name).count,
framework.db.vulns(workspace: ws.name).count,
framework.db.creds(workspace: ws.name).count,
framework.db.loots(workspace: ws.name).count,
framework.db.notes(workspace: ws.name).count
]
end
@ -310,7 +310,7 @@ class Db
each_host_range_chunk(host_ranges) do |host_search|
break if !host_search.nil? && host_search.empty?
framework.db.hosts(framework.db.workspace, false, host_search).each do |host|
framework.db.hosts(address: host_search).each do |host|
framework.db.update_host(host_data.merge(id: host.id))
framework.db.report_note(host: host.address, type: "host.#{attribute}", data: host_data[attribute])
end
@ -561,7 +561,7 @@ class Db
each_host_range_chunk(host_ranges) do |host_search|
break if !host_search.nil? && host_search.empty?
framework.db.hosts(framework.db.workspace, onlyup, host_search, search_term = search_term).each do |host|
framework.db.hosts(address: host_search, non_dead: onlyup, search_term: search_term).each do |host|
matched_host_ids << host.id
columns = col_names.map do |n|
# Deal with the special cases
@ -776,9 +776,10 @@ class Db
each_host_range_chunk(host_ranges) do |host_search|
break if !host_search.nil? && host_search.empty?
opts[:addresses] = host_search
opts[:workspace] = framework.db.workspace
opts[:hosts] = {address: host_search} if !host_search.nil?
opts[:port] = ports if ports
framework.db.services(framework.db.workspace, opts).each do |service|
framework.db.services(opts).each do |service|
host = service.host
matched_service_ids << service.id
@ -1311,12 +1312,12 @@ class Db
matched_loot_ids = []
loots = []
if host_ranges.compact.empty?
loots = loots + framework.db.loots(framework.db.workspace, {:search_term => search_term})
loots = loots + framework.db.loots(workspace: framework.db.workspace, search_term: search_term)
else
each_host_range_chunk(host_ranges) do |host_search|
break if !host_search.nil? && host_search.empty?
loots = loots + framework.db.loots(framework.db.workspace, { :hosts => { :address => host_search }, :search_term => search_term })
loots = loots + framework.db.loots(workspace: framework.db.workspace, hosts: { address: host_search }, search_term: search_term)
end
end

View File

@ -321,7 +321,7 @@ class MetasploitModule < Msf::Auxiliary
end
def existing_loot(ltype, key_id)
framework.db.loots(myworkspace).where(ltype: ltype).select {|l| l.info == key_id}.first
framework.db.loots(workspace: myworkspace).where(ltype: ltype).select {|l| l.info == key_id}.first
end
def store_public_keyfile(ip,user,key_id,key_data)

View File

@ -1062,7 +1062,7 @@ module Msf
return
end
targets = ""
framework.db.hosts(framework.db.workspace).each do |host|
framework.db.hosts.each do |host|
targets << host.address
targets << ","
end

View File

@ -6,7 +6,6 @@ RSpec.shared_examples_for 'Msf::DBManager::Host' do
it { is_expected.to respond_to :has_host? }
end
it { is_expected.to respond_to :find_or_create_host }
it { is_expected.to respond_to :get_host }
it { is_expected.to respond_to :hosts }

View File

@ -1,10 +1,9 @@
RSpec.shared_examples_for 'Msf::DBManager::Note' do
if ENV['REMOTE_DB']
before {skip("Awaiting port")}
unless ENV['REMOTE_DB']
it { is_expected.to respond_to :each_note }
end
it { is_expected.to respond_to :each_note }
it { is_expected.to respond_to :find_or_create_note }
it { is_expected.to respond_to :notes }
it { is_expected.to respond_to :report_note }

View File

@ -1,12 +1,12 @@
RSpec.shared_examples_for 'Msf::DBManager::Service' do
it { is_expected.to respond_to :delete_service }
unless ENV['REMOTE_DB']
it { is_expected.to respond_to :delete_service }
it { is_expected.to respond_to :each_service }
it { is_expected.to respond_to :find_or_create_service }
it { is_expected.to respond_to :get_service }
end
it { is_expected.to respond_to :report_service }
it { is_expected.to respond_to :find_or_create_service }
it { is_expected.to respond_to :services }
it { is_expected.to respond_to :report_service }
end