Land #9220, Module cache improvements
commit
7fe237abe1
|
@ -93,3 +93,7 @@ docker-compose.local*
|
|||
# Ignore python bytecode
|
||||
*.pyc
|
||||
rspec.failures
|
||||
|
||||
|
||||
#Ignore any base disk store files
|
||||
db/modules_metadata_base.pstore
|
Binary file not shown.
|
@ -22,6 +22,7 @@ module Metasploit::Framework::Spec::Constants
|
|||
Error
|
||||
External
|
||||
Loader
|
||||
Metadata
|
||||
MetasploitClassCompatibilityError
|
||||
Namespace
|
||||
VersionCompatibilityError
|
||||
|
|
|
@ -233,24 +233,8 @@ class Framework
|
|||
}
|
||||
end
|
||||
|
||||
# TODO: Anything still using this should be ported to use metadata::cache search
|
||||
def search(match, logger: nil)
|
||||
# Check if the database is usable
|
||||
use_db = true
|
||||
if self.db
|
||||
if !(self.db.migrated && self.db.modules_cached)
|
||||
logger.print_warning("Module database cache not built yet, using slow search") if logger
|
||||
use_db = false
|
||||
end
|
||||
else
|
||||
logger.print_warning("Database not connected, using slow search") if logger
|
||||
use_db = false
|
||||
end
|
||||
|
||||
# Used the database for search
|
||||
if use_db
|
||||
return self.db.search_modules(match)
|
||||
end
|
||||
|
||||
# Do an in-place search
|
||||
matches = []
|
||||
[ self.exploits, self.auxiliary, self.post, self.payloads, self.nops, self.encoders ].each do |mset|
|
||||
|
|
|
@ -21,7 +21,7 @@ module Msf::Module::FullName
|
|||
#
|
||||
|
||||
def fullname
|
||||
type + '/' + refname
|
||||
"#{type}/#{refname}"
|
||||
end
|
||||
|
||||
def promptname
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
# Gems
|
||||
#
|
||||
require 'active_support/concern'
|
||||
require 'msf/core/modules/metadata/cache'
|
||||
|
||||
# Concerns the module cache maintained by the {Msf::ModuleManager}.
|
||||
module Msf::ModuleManager::Cache
|
||||
|
@ -98,7 +99,7 @@ module Msf::ModuleManager::Cache
|
|||
end
|
||||
|
||||
# @overload refresh_cache_from_module_files
|
||||
# Rebuilds database and in-memory cache for all modules.
|
||||
# Rebuilds module metadata store and in-memory cache for all modules.
|
||||
#
|
||||
# @return [void]
|
||||
# @overload refresh_cache_from_module_files(module_class_or_instance)
|
||||
|
@ -107,14 +108,21 @@ module Msf::ModuleManager::Cache
|
|||
# @param (see Msf::DBManager#update_module_details)
|
||||
# @return [void]
|
||||
def refresh_cache_from_module_files(module_class_or_instance = nil)
|
||||
if framework_migrated?
|
||||
if module_class_or_instance
|
||||
framework.db.update_module_details(module_class_or_instance)
|
||||
else
|
||||
framework.db.update_all_module_details
|
||||
end
|
||||
refresh_cache_from_database(self.module_paths)
|
||||
if module_class_or_instance
|
||||
Msf::Modules::Metadata::Cache.instance.refresh_metadata_instance(module_class_or_instance)
|
||||
else
|
||||
module_sets =
|
||||
[
|
||||
['exploit', @framework.exploits],
|
||||
['auxiliary', @framework.auxiliary],
|
||||
['post', @framework.post],
|
||||
['payload', @framework.payloads],
|
||||
['encoder', @framework.encoders],
|
||||
['nop', @framework.nops]
|
||||
]
|
||||
Msf::Modules::Metadata::Cache.instance.refresh_metadata(module_sets)
|
||||
end
|
||||
refresh_cache_from_database(self.module_paths)
|
||||
end
|
||||
|
||||
# Refreshes the in-memory cache from the database cache.
|
||||
|
@ -126,19 +134,11 @@ module Msf::ModuleManager::Cache
|
|||
|
||||
protected
|
||||
|
||||
# Returns whether the framework migrations have been run already.
|
||||
#
|
||||
# @return [true] if migrations have been run
|
||||
# @return [false] otherwise
|
||||
def framework_migrated?
|
||||
framework.db && framework.db.migrated
|
||||
end
|
||||
|
||||
# @!attribute [rw] module_info_by_path
|
||||
# @return (see #module_info_by_path_from_database!)
|
||||
# @return (see #module_info_by_path_from_store!)
|
||||
attr_accessor :module_info_by_path
|
||||
|
||||
# Return a module info from Mdm::Module::Details in database.
|
||||
# Return a module info from Msf::Modules::Metadata::Obj.
|
||||
#
|
||||
# @note Also sets module_set(module_type)[module_reference_name] to Msf::SymbolicModule if it is not already set.
|
||||
#
|
||||
|
@ -148,41 +148,35 @@ module Msf::ModuleManager::Cache
|
|||
def module_info_by_path_from_database!(allowed_paths=[""])
|
||||
self.module_info_by_path = {}
|
||||
|
||||
if framework_migrated?
|
||||
allowed_paths = allowed_paths.map{|x| x + "/"}
|
||||
allowed_paths = allowed_paths.map{|x| x + "/"}
|
||||
|
||||
ActiveRecord::Base.connection_pool.with_connection do
|
||||
# TODO record module parent_path in Mdm::Module::Detail so it does not need to be derived from file.
|
||||
# Use find_each so Mdm::Module::Details are returned in batches, which will
|
||||
# handle the growing number of modules better than all.each.
|
||||
Mdm::Module::Detail.find_each do |module_detail|
|
||||
path = module_detail.file
|
||||
type = module_detail.mtype
|
||||
reference_name = module_detail.refname
|
||||
metadata = Msf::Modules::Metadata::Cache.instance.get_metadata
|
||||
metadata.each do |module_metadata|
|
||||
path = module_metadata.path
|
||||
type = module_metadata.type
|
||||
reference_name = module_metadata.ref_name
|
||||
|
||||
# Skip cached modules that are not in our allowed load paths
|
||||
next if allowed_paths.select{|x| path.index(x) == 0}.empty?
|
||||
# Skip cached modules that are not in our allowed load paths
|
||||
next if allowed_paths.select{|x| path.index(x) == 0}.empty?
|
||||
|
||||
# The load path is assumed to be the next level above the type directory
|
||||
type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
|
||||
parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
|
||||
# The load path is assumed to be the next level above the type directory
|
||||
type_dir = File.join('', Mdm::Module::Detail::DIRECTORY_BY_TYPE[type], '')
|
||||
parent_path = path.split(type_dir)[0..-2].join(type_dir) # TODO: rewrite
|
||||
|
||||
module_info_by_path[path] = {
|
||||
:reference_name => reference_name,
|
||||
:type => type,
|
||||
:parent_path => parent_path,
|
||||
:modification_time => module_detail.mtime
|
||||
}
|
||||
module_info_by_path[path] = {
|
||||
:reference_name => reference_name,
|
||||
:type => type,
|
||||
:parent_path => parent_path,
|
||||
:modification_time => module_metadata.mod_time
|
||||
}
|
||||
|
||||
typed_module_set = module_set(type)
|
||||
typed_module_set = module_set(type)
|
||||
|
||||
# Don't want to trigger as {Msf::ModuleSet#create} so check for
|
||||
# key instead of using ||= which would call {Msf::ModuleSet#[]}
|
||||
# which would potentially call {Msf::ModuleSet#create}.
|
||||
unless typed_module_set.has_key? reference_name
|
||||
typed_module_set[reference_name] = Msf::SymbolicModule
|
||||
end
|
||||
end
|
||||
# Don't want to trigger as {Msf::ModuleSet#create} so check for
|
||||
# key instead of using ||= which would call {Msf::ModuleSet#[]}
|
||||
# which would potentially call {Msf::ModuleSet#create}.
|
||||
unless typed_module_set.has_key? reference_name
|
||||
typed_module_set[reference_name] = Msf::SymbolicModule
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
# -*- coding: binary -*-
|
||||
require 'msf/core/modules'
|
||||
|
||||
# Namespace for module metadata related data and operations
|
||||
module Msf::Modules::Metadata
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
require 'singleton'
|
||||
require 'msf/events'
|
||||
require 'rex/ui/text/output/stdio'
|
||||
require 'msf/core/constants'
|
||||
require 'msf/core/modules/metadata'
|
||||
require 'msf/core/modules/metadata/obj'
|
||||
require 'msf/core/modules/metadata/search'
|
||||
require 'msf/core/modules/metadata/store'
|
||||
|
||||
#
|
||||
# Core service class that provides storage of module metadata as well as operations on the metadata.
|
||||
# Note that operations on this metadata are included as separate modules.
|
||||
#
|
||||
module Msf
|
||||
module Modules
|
||||
module Metadata
|
||||
|
||||
class Cache
|
||||
include Singleton
|
||||
include Msf::Modules::Metadata::Search
|
||||
include Msf::Modules::Metadata::Store
|
||||
|
||||
#
|
||||
# Refreshes cached module metadata as well as updating the store
|
||||
#
|
||||
def refresh_metadata_instance(module_instance)
|
||||
refresh_metadata_instance_internal(module_instance)
|
||||
update_store
|
||||
end
|
||||
|
||||
#
|
||||
# Returns the module data cache, but first ensures all the metadata is loaded
|
||||
#
|
||||
def get_metadata
|
||||
wait_for_load
|
||||
@module_metadata_cache.values
|
||||
end
|
||||
|
||||
#
|
||||
# Checks for modules loaded that are not a part of the cache and updates the underlying store
|
||||
# if there are changes.
|
||||
#
|
||||
def refresh_metadata(module_sets)
|
||||
unchanged_module_references = get_unchanged_module_references
|
||||
has_changes = false
|
||||
module_sets.each do |mt|
|
||||
unchanged_reference_name_set = unchanged_module_references[mt[0]]
|
||||
|
||||
mt[1].keys.sort.each do |mn|
|
||||
next if unchanged_reference_name_set.include? mn
|
||||
module_instance = mt[1].create(mn)
|
||||
next if not module_instance
|
||||
begin
|
||||
refresh_metadata_instance_internal(module_instance)
|
||||
has_changes = true
|
||||
rescue Exception => e
|
||||
elog("Error updating module details for #{module_instance.fullname}: #{$!.class} #{$!} : #{e.message}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
update_store if has_changes
|
||||
end
|
||||
|
||||
#
|
||||
# Returns a hash(type->set) which references modules that have not changed.
|
||||
#
|
||||
def get_unchanged_module_references
|
||||
skip_reference_name_set_by_module_type = Hash.new { |hash, module_type|
|
||||
hash[module_type] = Set.new
|
||||
}
|
||||
|
||||
@module_metadata_cache.each_value do |module_metadata|
|
||||
|
||||
unless module_metadata.path and ::File.exist?(module_metadata.path)
|
||||
next
|
||||
end
|
||||
|
||||
if ::File.mtime(module_metadata.path).to_i != module_metadata.mod_time.to_i
|
||||
next
|
||||
end
|
||||
|
||||
skip_reference_name_set = skip_reference_name_set_by_module_type[module_metadata.type]
|
||||
skip_reference_name_set.add(module_metadata.ref_name)
|
||||
end
|
||||
|
||||
return skip_reference_name_set_by_module_type
|
||||
end
|
||||
|
||||
#######
|
||||
private
|
||||
#######
|
||||
|
||||
def wait_for_load
|
||||
@load_thread.join unless @store_loaded
|
||||
end
|
||||
|
||||
def refresh_metadata_instance_internal(module_instance)
|
||||
metadata_obj = Obj.new(module_instance)
|
||||
@module_metadata_cache[get_cache_key(module_instance)] = metadata_obj
|
||||
end
|
||||
|
||||
def get_cache_key(module_instance)
|
||||
key = ''
|
||||
key << (module_instance.type.nil? ? '' : module_instance.type)
|
||||
key << '_'
|
||||
key << module_instance.refname
|
||||
return key
|
||||
end
|
||||
|
||||
def initialize
|
||||
@module_metadata_cache = {}
|
||||
@store_loaded = false
|
||||
@console = Rex::Ui::Text::Output::Stdio.new
|
||||
@load_thread = Thread.new {
|
||||
init_store
|
||||
@store_loaded = true
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,71 @@
|
|||
require 'msf/core/modules/metadata'
|
||||
|
||||
#
|
||||
# Simple object for storing a modules metadata.
|
||||
#
|
||||
module Msf
|
||||
module Modules
|
||||
module Metadata
|
||||
|
||||
class Obj
|
||||
attr_reader :name
|
||||
attr_reader :full_name
|
||||
attr_reader :rank
|
||||
attr_reader :disclosure_date
|
||||
attr_reader :type
|
||||
attr_reader :author
|
||||
attr_reader :description
|
||||
attr_reader :references
|
||||
attr_reader :is_server
|
||||
attr_reader :is_client
|
||||
attr_reader :platform
|
||||
attr_reader :arch
|
||||
attr_reader :rport
|
||||
attr_reader :targets
|
||||
attr_reader :mod_time
|
||||
attr_reader :is_install_path
|
||||
attr_reader :ref_name
|
||||
|
||||
def initialize(module_instance)
|
||||
@name = module_instance.name
|
||||
@full_name = module_instance.fullname
|
||||
@disclosure_date = module_instance.disclosure_date
|
||||
@rank = module_instance.rank.to_i
|
||||
@type = module_instance.type
|
||||
@description = module_instance.description.to_s.strip
|
||||
@author = module_instance.author.map{|x| x.to_s}
|
||||
@references = module_instance.references.map{|x| [x.ctx_id, x.ctx_val].join("-") }
|
||||
@is_server = (module_instance.respond_to?(:stance) and module_instance.stance == "aggressive")
|
||||
@is_client = (module_instance.respond_to?(:stance) and module_instance.stance == "passive")
|
||||
@platform = module_instance.platform_to_s
|
||||
@arch = module_instance.arch_to_s
|
||||
@rport = module_instance.datastore['RPORT'].to_s
|
||||
@path = module_instance.file_path
|
||||
@mod_time = ::File.mtime(@path) rescue Time.now
|
||||
@ref_name = module_instance.refname
|
||||
install_path = Msf::Config.install_root.to_s
|
||||
if (@path.to_s.include? (install_path))
|
||||
@path = @path.sub(install_path, '')
|
||||
@is_install_path = true
|
||||
end
|
||||
|
||||
if module_instance.respond_to?(:targets) and module_instance.targets
|
||||
@targets = module_instance.targets.map{|x| x.name}
|
||||
end
|
||||
end
|
||||
|
||||
def update_mod_time(mod_time)
|
||||
@mod_time = mod_time
|
||||
end
|
||||
|
||||
def path
|
||||
if @is_install_path
|
||||
return ::File.join(Msf::Config.install_root, @path)
|
||||
end
|
||||
|
||||
@path
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,120 @@
|
|||
require 'msf/core/modules/metadata'
|
||||
|
||||
#
|
||||
# Provides search operations on the module metadata cache.
|
||||
#
|
||||
module Msf::Modules::Metadata::Search
|
||||
#
|
||||
# Searches the module metadata using the passed search string.
|
||||
#
|
||||
def find(search_string)
|
||||
search_results = []
|
||||
|
||||
get_metadata.each { |module_metadata|
|
||||
if is_match(search_string, module_metadata)
|
||||
search_results << module_metadata
|
||||
end
|
||||
}
|
||||
|
||||
return search_results
|
||||
end
|
||||
|
||||
#######
|
||||
private
|
||||
#######
|
||||
|
||||
def is_match(search_string, module_metadata)
|
||||
return false if not search_string
|
||||
|
||||
search_string += ' '
|
||||
|
||||
# Split search terms by space, but allow quoted strings
|
||||
terms = search_string.split(/\"/).collect{|t| t.strip==t ? t : t.split(' ')}.flatten
|
||||
terms.delete('')
|
||||
|
||||
# All terms are either included or excluded
|
||||
res = {}
|
||||
|
||||
terms.each do |t|
|
||||
f,v = t.split(":", 2)
|
||||
if not v
|
||||
v = f
|
||||
f = 'text'
|
||||
end
|
||||
next if v.length == 0
|
||||
f.downcase!
|
||||
v.downcase!
|
||||
res[f] ||=[ [], [] ]
|
||||
if v[0,1] == "-"
|
||||
next if v.length == 1
|
||||
res[f][1] << v[1,v.length-1]
|
||||
else
|
||||
res[f][0] << v
|
||||
end
|
||||
end
|
||||
|
||||
k = res
|
||||
|
||||
[0,1].each do |mode|
|
||||
match = false
|
||||
k.keys.each do |t|
|
||||
next if k[t][mode].length == 0
|
||||
|
||||
k[t][mode].each do |w|
|
||||
# Reset the match flag for each keyword for inclusive search
|
||||
match = false if mode == 0
|
||||
|
||||
# Convert into a case-insensitive regex
|
||||
r = Regexp.new(Regexp.escape(w), true)
|
||||
|
||||
case t
|
||||
when 'text'
|
||||
terms = [module_metadata.name, module_metadata.full_name, module_metadata.description] + module_metadata.references + module_metadata.author
|
||||
|
||||
if module_metadata.targets
|
||||
terms = terms + module_metadata.targets
|
||||
end
|
||||
match = [t,w] if terms.any? { |x| x =~ r }
|
||||
when 'name'
|
||||
match = [t,w] if module_metadata.name =~ r
|
||||
when 'path'
|
||||
match = [t,w] if module_metadata.full_name =~ r
|
||||
when 'author'
|
||||
match = [t,w] if module_metadata.author.any? { |a| a =~ r }
|
||||
when 'os', 'platform'
|
||||
match = [t,w] if module_metadata.platform =~ r or module_metadata.arch =~ r
|
||||
|
||||
if module_metadata.targets
|
||||
match = [t,w] if module_metadata.targets.any? { |t| t =~ r }
|
||||
end
|
||||
when 'port'
|
||||
match = [t,w] if module_metadata.rport =~ r
|
||||
when 'type'
|
||||
match = [t,w] if Msf::MODULE_TYPES.any? { |modt| w == modt and module_metadata.type == modt }
|
||||
when 'app'
|
||||
match = [t,w] if (w == "server" and module_metadata.is_server)
|
||||
match = [t,w] if (w == "client" and module_metadata.is_client)
|
||||
when 'cve'
|
||||
match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^cve\-/i and ref =~ r }
|
||||
when 'bid'
|
||||
match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^bid\-/i and ref =~ r }
|
||||
when 'edb'
|
||||
match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^edb\-/i and ref =~ r }
|
||||
end
|
||||
break if match
|
||||
end
|
||||
# Filter this module if no matches for a given keyword type
|
||||
if mode == 0 and not match
|
||||
return false
|
||||
end
|
||||
end
|
||||
# Filter this module if we matched an exclusion keyword (-value)
|
||||
if mode == 1 and match
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
true
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
require 'pstore'
|
||||
require 'msf/core/modules/metadata'
|
||||
|
||||
#
|
||||
# Handles storage of module metadata on disk. A base metadata file is always included - this was added to ensure a much
|
||||
# better first time user experience as generating the user based metadata file requires 100+ mb at the time of creating
|
||||
# this module. Subsequent starts of metasploit will load from a user specific metadata file as users potentially load modules
|
||||
# from other places.
|
||||
#
|
||||
module Msf::Modules::Metadata::Store
|
||||
|
||||
BaseMetaDataFile = 'modules_metadata_base.pstore'
|
||||
UserMetaDataFile = 'modules_metadata.pstore'
|
||||
|
||||
#
|
||||
# Initializes from user store (under ~/store/.msf4) if it exists. else base file (under $INSTALL_ROOT/db) is copied and loaded.
|
||||
#
|
||||
def init_store
|
||||
load_metadata
|
||||
end
|
||||
|
||||
#
|
||||
# Update the module meta cache disk store
|
||||
#
|
||||
def update_store
|
||||
begin
|
||||
@store.transaction do
|
||||
@store[:module_metadata] = @module_metadata_cache
|
||||
end
|
||||
rescue Excepion => e
|
||||
elog("Unable to update metadata store: #{e.message}")
|
||||
end
|
||||
end
|
||||
|
||||
#######
|
||||
private
|
||||
#######
|
||||
|
||||
def load_metadata
|
||||
begin
|
||||
retries ||= 0
|
||||
copied = configure_user_store
|
||||
@store = PStore.new(@path_to_user_metadata, true)
|
||||
@module_metadata_cache = @store.transaction(true) { @store[:module_metadata]}
|
||||
validate_data(copied) if (!@module_metadata_cache.nil? && @module_metadata_cache.size > 0)
|
||||
@module_metadata_cache = {} if @module_metadata_cache.nil?
|
||||
rescue Exception => e
|
||||
retries +=1
|
||||
|
||||
# Try to handle the scenario where the file is corrupted
|
||||
if (retries < 2 && ::File.exist?(@path_to_user_metadata))
|
||||
elog('Possible corrupt user metadata store, attempting restore')
|
||||
FileUtils.remove(@path_to_user_metadata)
|
||||
retry
|
||||
else
|
||||
@console.print_warning('Unable to load module metadata from disk see error log')
|
||||
elog("Unable to load module metadata: #{e.message}")
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
def validate_data(copied)
|
||||
size_prior = @module_metadata_cache.size
|
||||
@module_metadata_cache.delete_if {|key, module_metadata| !::File.exist?(module_metadata.path)}
|
||||
|
||||
if (copied)
|
||||
@module_metadata_cache.each_value {|module_metadata|
|
||||
module_metadata.update_mod_time(::File.mtime(module_metadata.path))
|
||||
}
|
||||
end
|
||||
|
||||
update_store if (size_prior != @module_metadata_cache.size || copied)
|
||||
end
|
||||
|
||||
def configure_user_store
|
||||
copied = false
|
||||
@path_to_user_metadata = get_user_store
|
||||
path_to_base_metadata = ::File.join(Msf::Config.install_root, "db", BaseMetaDataFile)
|
||||
user_file_exists = ::File.exist?(@path_to_user_metadata)
|
||||
base_file_exists = ::File.exist?(path_to_base_metadata)
|
||||
|
||||
if (!base_file_exists)
|
||||
wlog("Missing base module metadata file: #{path_to_base_metadata}")
|
||||
return copied if !user_file_exists
|
||||
end
|
||||
|
||||
if (!user_file_exists)
|
||||
FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
|
||||
copied = true
|
||||
|
||||
dlog('Created user based module store')
|
||||
|
||||
# Update the user based module store if an updated base file is created/pushed
|
||||
elsif (::File.mtime(path_to_base_metadata).to_i > ::File.mtime(@path_to_user_metadata).to_i)
|
||||
FileUtils.remove(@path_to_user_metadata)
|
||||
FileUtils.cp(path_to_base_metadata, @path_to_user_metadata)
|
||||
copied = true
|
||||
dlog('Updated user based module store')
|
||||
end
|
||||
|
||||
return copied
|
||||
end
|
||||
|
||||
def get_user_store
|
||||
store_dir = ::File.join(Msf::Config.config_directory, "store")
|
||||
FileUtils.mkdir(store_dir) if !::File.exist?(store_dir)
|
||||
return ::File.join(store_dir, UserMetaDataFile)
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -418,12 +418,12 @@ module Msf
|
|||
|
||||
# Display the table of matches
|
||||
tbl = generate_module_table("Matching Modules", search_term)
|
||||
framework.search(match, logger: self).each do |m|
|
||||
Msf::Modules::Metadata::Cache.instance.find(match).each do |m|
|
||||
tbl << [
|
||||
m.fullname,
|
||||
m.disclosure_date.nil? ? "" : m.disclosure_date.strftime("%Y-%m-%d"),
|
||||
RankingName[m.rank].to_s,
|
||||
m.name
|
||||
m.full_name,
|
||||
m.disclosure_date.nil? ? '' : m.disclosure_date.strftime("%Y-%m-%d"),
|
||||
RankingName[m.rank].to_s,
|
||||
m.name
|
||||
]
|
||||
end
|
||||
print_line(tbl.to_s)
|
||||
|
|
|
@ -196,7 +196,7 @@ class Driver < Msf::Ui::Driver
|
|||
self.framework.init_module_paths(module_paths: opts['ModulePath'])
|
||||
end
|
||||
|
||||
if framework.db.active && !opts['DeferModuleLoads']
|
||||
if !opts['DeferModuleLoads']
|
||||
framework.threads.spawn("ModuleCacheRebuild", true) do
|
||||
framework.modules.refresh_cache_from_module_files
|
||||
end
|
||||
|
|
|
@ -1,8 +1,18 @@
|
|||
RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
|
||||
|
||||
# Wait for data to be loaded
|
||||
before(:all) do
|
||||
Msf::Modules::Metadata::Cache.instance.get_metadata
|
||||
end
|
||||
|
||||
let(:parent_path) do
|
||||
parent_pathname.to_path
|
||||
end
|
||||
|
||||
let(:metadata_cache) do
|
||||
Msf::Modules::Metadata::Cache.instance
|
||||
end
|
||||
|
||||
let(:parent_pathname) do
|
||||
Metasploit::Framework.root.join('modules')
|
||||
end
|
||||
|
@ -221,73 +231,37 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
|
|||
end
|
||||
|
||||
context '#refresh_cache_from_module_files' do
|
||||
before(:example) do
|
||||
allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
|
||||
end
|
||||
|
||||
context 'with framework migrated' do
|
||||
let(:framework_migrated?) do
|
||||
true
|
||||
context 'with module argument' do
|
||||
def refresh_cache_from_module_files
|
||||
module_manager.refresh_cache_from_module_files(module_class_or_instance)
|
||||
end
|
||||
|
||||
context 'with module argument' do
|
||||
def refresh_cache_from_module_files
|
||||
module_manager.refresh_cache_from_module_files(module_class_or_instance)
|
||||
end
|
||||
|
||||
let(:module_class_or_instance) do
|
||||
Class.new(Msf::Module)
|
||||
end
|
||||
|
||||
it 'should update database and then update in-memory cache from the database for the given module_class_or_instance' do
|
||||
expect(framework.db).to receive(:update_module_details).with(module_class_or_instance).ordered
|
||||
expect(module_manager).to receive(:refresh_cache_from_database).ordered
|
||||
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
let(:module_class_or_instance) do
|
||||
Class.new(Msf::Module)
|
||||
end
|
||||
|
||||
context 'without module argument' do
|
||||
def refresh_cache_from_module_files
|
||||
module_manager.refresh_cache_from_module_files
|
||||
end
|
||||
it 'should update store and then update in-memory cache from the store for the given module_class_or_instance' do
|
||||
expect(metadata_cache).to receive(:refresh_metadata_instance).with(module_class_or_instance).ordered
|
||||
expect(module_manager).to receive(:refresh_cache_from_database).ordered
|
||||
|
||||
it 'should update database and then update in-memory cache from the database for all modules' do
|
||||
expect(framework.db).to receive(:update_all_module_details).ordered
|
||||
expect(module_manager).to receive(:refresh_cache_from_database)
|
||||
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
end
|
||||
|
||||
context 'without framework migrated' do
|
||||
context 'without module argument' do
|
||||
def refresh_cache_from_module_files
|
||||
module_manager.refresh_cache_from_module_files
|
||||
end
|
||||
|
||||
let(:framework_migrated?) do
|
||||
false
|
||||
end
|
||||
|
||||
it 'should not call Msf::DBManager#update_module_details' do
|
||||
expect(framework.db).not_to receive(:update_module_details)
|
||||
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
|
||||
it 'should not call Msf::DBManager#update_all_module_details' do
|
||||
expect(framework.db).not_to receive(:update_all_module_details)
|
||||
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
|
||||
it 'should not call #refresh_cache_from_database' do
|
||||
expect(module_manager).not_to receive(:refresh_cache_from_database)
|
||||
it 'should update store and then update in-memory cache from the store for all modules' do
|
||||
expect(metadata_cache).to receive(:refresh_metadata).ordered
|
||||
expect(module_manager).to receive(:refresh_cache_from_database)
|
||||
|
||||
refresh_cache_from_module_files
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
context '#refresh_cache_from_database' do
|
||||
|
@ -302,41 +276,6 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
|
|||
end
|
||||
end
|
||||
|
||||
context '#framework_migrated?' do
|
||||
subject(:framework_migrated?) do
|
||||
module_manager.send(:framework_migrated?)
|
||||
end
|
||||
|
||||
context 'with framework database' do
|
||||
before(:example) do
|
||||
expect(framework.db).to receive(:migrated).and_return(migrated)
|
||||
end
|
||||
|
||||
context 'with migrated' do
|
||||
let(:migrated) do
|
||||
true
|
||||
end
|
||||
|
||||
it { is_expected.to be_truthy }
|
||||
end
|
||||
|
||||
context 'without migrated' do
|
||||
let(:migrated) do
|
||||
false
|
||||
end
|
||||
|
||||
it { is_expected.to be_falsey }
|
||||
end
|
||||
end
|
||||
|
||||
context 'without framework database' do
|
||||
before(:example) do
|
||||
expect(framework).to receive(:db).and_return(nil)
|
||||
end
|
||||
|
||||
it { is_expected.to be_falsey }
|
||||
end
|
||||
end
|
||||
|
||||
context '#module_info_by_path' do
|
||||
it 'should have protected method module_info_by_path' do
|
||||
|
@ -359,101 +298,78 @@ RSpec.shared_examples_for 'Msf::ModuleManager::Cache' do
|
|||
module_manager.send(:module_info_by_path_from_database!)
|
||||
end
|
||||
|
||||
before(:example) do
|
||||
allow(module_manager).to receive(:framework_migrated?).and_return(framework_migrated?)
|
||||
it 'should call get metadata' do
|
||||
allow(metadata_cache).to receive(:get_metadata).and_return([])
|
||||
expect(metadata_cache).to receive(:get_metadata)
|
||||
|
||||
module_info_by_path_from_database!
|
||||
end
|
||||
|
||||
context 'with framework migrated' do
|
||||
let(:framework_migrated?) do
|
||||
true
|
||||
context 'with database cache' do
|
||||
#
|
||||
# Let!s (let + before(:each))
|
||||
#
|
||||
|
||||
let!(:mdm_module_detail) do
|
||||
FactoryGirl.create(:mdm_module_detail,
|
||||
:file => path,
|
||||
:mtype => type,
|
||||
:mtime => pathname.mtime,
|
||||
:refname => reference_name
|
||||
)
|
||||
end
|
||||
|
||||
it 'should use ActiveRecord::Batches#find_each to enumerate Mdm::Module::Details in batches' do
|
||||
expect(Mdm::Module::Detail).to receive(:find_each)
|
||||
|
||||
it 'should create cache entry for path' do
|
||||
module_info_by_path_from_database!
|
||||
|
||||
expect(module_info_by_path).to have_key(path)
|
||||
end
|
||||
|
||||
context 'with database cache' do
|
||||
#
|
||||
# Let!s (let + before(:each))
|
||||
#
|
||||
|
||||
let!(:mdm_module_detail) do
|
||||
FactoryGirl.create(:mdm_module_detail,
|
||||
:file => path,
|
||||
:mtype => type,
|
||||
:mtime => pathname.mtime,
|
||||
:refname => reference_name
|
||||
)
|
||||
context 'cache entry' do
|
||||
subject(:cache_entry) do
|
||||
module_info_by_path[path]
|
||||
end
|
||||
|
||||
it 'should create cache entry for path' do
|
||||
before(:example) do
|
||||
module_info_by_path_from_database!
|
||||
|
||||
expect(module_info_by_path).to have_key(path)
|
||||
end
|
||||
|
||||
context 'cache entry' do
|
||||
subject(:cache_entry) do
|
||||
module_info_by_path[path]
|
||||
end
|
||||
it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
|
||||
it { expect(subject[:parent_path]).to eq(parent_path) }
|
||||
it { expect(subject[:reference_name]).to eq(reference_name) }
|
||||
it { expect(subject[:type]).to eq(type) }
|
||||
end
|
||||
|
||||
context 'typed module set' do
|
||||
let(:typed_module_set) do
|
||||
module_manager.module_set(type)
|
||||
end
|
||||
|
||||
context 'with reference_name' do
|
||||
before(:example) do
|
||||
module_info_by_path_from_database!
|
||||
typed_module_set[reference_name] = double('Msf::Module')
|
||||
end
|
||||
|
||||
it { expect(subject[:modification_time]).to be_within(1.second).of(pathname_modification_time) }
|
||||
it { expect(subject[:parent_path]).to eq(parent_path) }
|
||||
it { expect(subject[:reference_name]).to eq(reference_name) }
|
||||
it { expect(subject[:type]).to eq(type) }
|
||||
end
|
||||
|
||||
context 'typed module set' do
|
||||
let(:typed_module_set) do
|
||||
module_manager.module_set(type)
|
||||
end
|
||||
|
||||
context 'with reference_name' do
|
||||
before(:example) do
|
||||
typed_module_set[reference_name] = double('Msf::Module')
|
||||
end
|
||||
|
||||
it 'should not change reference_name value' do
|
||||
expect {
|
||||
module_info_by_path_from_database!
|
||||
}.to_not change {
|
||||
typed_module_set[reference_name]
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
context 'without reference_name' do
|
||||
it 'should set reference_name value to Msf::SymbolicModule' do
|
||||
it 'should not change reference_name value' do
|
||||
expect {
|
||||
module_info_by_path_from_database!
|
||||
}.to_not change {
|
||||
typed_module_set[reference_name]
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# have to use fetch because [] will trigger de-symbolization and
|
||||
# instantiation.
|
||||
expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
|
||||
end
|
||||
context 'without reference_name' do
|
||||
it 'should set reference_name value to Msf::SymbolicModule' do
|
||||
module_info_by_path_from_database!
|
||||
|
||||
# have to use fetch because [] will trigger de-symbolization and
|
||||
# instantiation.
|
||||
expect(typed_module_set.fetch(reference_name)).to eq Msf::SymbolicModule
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'without framework migrated' do
|
||||
let(:framework_migrated?) do
|
||||
false
|
||||
end
|
||||
|
||||
it 'should reset #module_info_by_path' do
|
||||
# pre-fill module_info_by_path so change can be detected
|
||||
module_manager.send(:module_info_by_path=, double('In-memory Cache'))
|
||||
|
||||
module_info_by_path_from_database!
|
||||
|
||||
expect(module_info_by_path).to be_empty
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue