Refactor Msf::ModuleManager

[Fixes #36737359]

Refactor Msf::ModuleManager into concerns so its easier to understand and
duplicate code can be made DRY.  The refactoring also ensures that when
loading from directories, Fastlibs, or reloading, the wrapper module will
always be named so that activesupport/dependencies will function.

.gitignore

@@ -1,9 +1,11 @@
 # Rubymine project directory
 .idea
+.yardoc
 # Mac OS X files
 .DS_Store
 data/meterpreter/ext_server_pivot.dll
 data/meterpreter/ext_server_pivot.x64.dll
+doc
 external/source/meterpreter/java/bin
 external/source/meterpreter/java/build
 external/source/meterpreter/java/extensions

.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p194@metasploit-framework"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.15.7 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

Gemfile

@@ -1,6 +1,10 @@
 source 'http://rubygems.org'
-gem 'rails', '3.2.2'
-gem 'metasploit_data_models', '0.0.2', :git => "git://github.com/rapid7/metasploit_data_models.git"
-gem 'pg', '>=0.13'
-gem 'msgpack'
-gem 'nokogiri'
+
+group :development do
+  # running documention generation tasks
+  gem 'rake'
+  # Markdown formatting for yara
+  gem 'redcarpet'
+  # generating documention
+  gem 'yard'
+end

Gemfile.lock

@@ -0,0 +1,14 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    rake (0.9.2.2)
+    redcarpet (2.1.1)
+    yard (0.8.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  redcarpet
+  yard

Rakefile

@@ -0,0 +1,43 @@
+require 'bundler/setup'
+
+require 'yard'
+
+namespace :yard do
+  yard_files = [
+      # Ruby source files first
+      'lib/**/*.rb',
+      # Anything after '-' is a normal documentation, not source
+      '-',
+      'COPYING',
+      'HACKING',
+      'THIRD-PARTY.md'
+  ]
+  yard_options = [
+      # don't generate documentation from the source of the gems in the gemcache.
+      '--exclude', 'lib/gemcache',
+      # include documentation for protected methods for developers extending the code.
+      '--protected'
+  ]
+
+  YARD::Rake::YardocTask.new(:doc) do |t|
+    t.files = yard_files
+    # --no-stats here as 'stats' task called after will print fuller stats
+    t.options = yard_options + ['--no-stats']
+
+    t.after = Proc.new {
+      Rake::Task['yard:stats'].execute
+    }
+  end
+
+  desc "Shows stats for YARD Documentation including listing undocumented modules, classes, constants, and methods"
+  task :stats => :environment do
+    stats = YARD::CLI::Stats.new
+    yard_arguments = yard_options + ['--compact', '--list-undoc'] + yard_files
+    stats.run *yard_arguments
+  end
+end
+
+# @todo Figure out how to just clone description from yard:doc
+desc "Generate YARD documentation"
+# allow calling namespace to as a task that goes to default task for namespace
+task :yard => ['yard:doc']

lib/msf/core.rb

@@ -35,6 +35,7 @@ require 'msf/core/framework'
 require 'msf/core/db_manager'
 require 'msf/core/event_dispatcher'
 require 'msf/core/module_manager'
+require 'msf/core/module_set'
 require 'msf/core/plugin_manager'
 require 'msf/core/session'
 require 'msf/core/session_manager'

lib/msf/core/module.rb

@@ -78,14 +78,6 @@ class Module
 		# The path from which the module was loaded.
 		#
 		attr_accessor :file_path
-
-		#
-		# Override the default Class#inspect which is useless for the way
-		# modules get loaded
-		#
-		def inspect
-			"#<Class for #{refname}>"
-		end
 	end
 
 	#

lib/msf/core/module_manager/cache.rb

@@ -0,0 +1,61 @@
+# Concerns the module cache maintained by the {Msf::ModuleManager}.
+module Msf::ModuleManager::Cache
+  extend ActiveSupport::Concern
+
+  attr_accessor :cache # :nodoc:
+
+  #
+  # Return a listing of all cached modules
+  #
+  def cache_entries
+    module_detail_by_file = {}
+
+    if framework_migrated?
+      ::Mdm::ModuleDetail.find(:all).each do |module_detail|
+        module_type = module_detail.mtype
+        refname = module_detail.refname
+
+        module_detail_by_file[module_detail.file] = {
+            :mtype => module_type,
+            :refname => refname,
+            :file => module_detail.file,
+            :mtime => module_detail.mtime
+        }
+
+        module_set(module_type)[refname] ||= SymbolicModule
+      end
+    end
+
+    module_detail_by_file
+  end
+
+  #
+  # Rebuild the cache for the module set
+  #
+  def rebuild_cache(mod = nil)
+    unless framework_migrated?
+      if mod
+        framework.db.update_module_details(mod)
+      else
+        framework.db.update_all_module_details
+      end
+
+      refresh_cache
+    end
+  end
+
+  def framework_migrated?
+    if framework.db and framework.db.migrated
+      true
+    else
+      false
+    end
+  end
+
+  #
+  # Reset the module cache
+  #
+  def refresh_cache
+    self.cache = cache_entries
+  end
+end

lib/msf/core/module_manager/loading.rb

@@ -0,0 +1,111 @@
+require 'msf/core/modules/loader/archive'
+require 'msf/core/modules/loader/directory'
+
+# Deals with loading modules for the {Msf::ModuleManager}
+module Msf::ModuleManager::Loading
+  extend ActiveSupport::Concern
+
+  #
+  # CONSTANTS
+  #
+
+  # Classes that can be used to load modules.
+  LOADER_CLASSES = [
+      Msf::Modules::Loader::Archive,
+      Msf::Modules::Loader::Directory
+  ]
+
+  #
+  # Returns the set of modules that failed to load.
+  #
+  def failed
+    return module_load_error_by_reference_name
+  end
+
+  def file_changed?(path)
+    changed = false
+
+    module_details = self.cache[path]
+
+    # if uncached then it counts as changed
+    # Payloads can't be cached due to stage/stager matching
+    if module_details.nil? or module_details[:mtype] == MODULE_PAYLOAD
+      changed = true
+    else
+      begin
+        current_modification_time = ::File.mtime(path).to_i
+      rescue ::Errno::ENOENT
+        # if the file does not exist now, that's a change
+        changed = true
+      else
+        cached_modification_time = module_details[:mtime].to_i
+
+        # if the file's modification time's different from the cache, then it's changed
+        if current_modification_time != cached_modification_time
+          changed = true
+        end
+      end
+    end
+
+    changed
+  end
+
+  attr_accessor :module_load_error_by_reference_name
+
+  # Called when a module is initially loaded such that it can be
+  # categorized accordingly.
+  #
+  def on_module_load(mod, type, name, modinfo)
+    # Payload modules require custom loading as the individual files
+    # may not directly contain a logical payload that a user would
+    # reference, such as would be the case with a payload stager or
+    # stage.  As such, when payload modules are loaded they are handed
+    # off to a special payload set.  The payload set, in turn, will
+    # automatically create all the permutations after all the payload
+    # modules have been loaded.
+
+    if (type != Msf::MODULE_PAYLOAD)
+      # Add the module class to the list of modules and add it to the
+      # type separated set of module classes
+      add_module(mod, name, modinfo)
+    end
+
+    module_set_by_type[type].add_module(mod, name, modinfo)
+  end
+
+  protected
+
+  # Return list of {LOADER_CLASSES} instances that load modules into this module manager
+  def loaders
+    unless instance_variable_defined? :@loaders
+      @loaders = LOADER_CLASSES.collect { |klass|
+        klass.new(self)
+      }
+    end
+
+    @loaders
+  end
+
+  # Load all of the modules from the supplied directory or archive
+  #
+  # @param [String] path Path to a directory or Fastlib archive
+  # @param [Hash] options
+  # @option options [Boolean] :force Whether the force loading the modules even if they are unchanged and already
+  #   loaded.
+  # @return [Hash{String => Integer}] Maps module type to number of modules loaded
+  def load_modules(path, options={})
+    options.assert_valid_keys(:force)
+
+    count_by_type = {}
+
+    loaders.each do |loader|
+      if loader.loadable?(path)
+        count_by_type = loader.load_modules(path, options)
+
+        break
+      end
+    end
+
+    count_by_type
+  end
+end

lib/msf/core/module_manager/module_paths.rb

@@ -0,0 +1,67 @@
+# Deals with module paths in the {Msf::ModuleManager}
+module Msf::ModuleManager::ModulePaths
+  extend ActiveSupport::Concern
+
+  # Adds a path to be searched for new modules.
+  #
+  # @param [String] path
+  # @return (see Msf::Modules::Loader::Base#load_modules)
+  def add_module_path(path)
+    nested_paths = []
+
+    if path =~ /\.fastlib$/
+      unless ::File.exist?(path)
+        raise RuntimeError, "The path supplied does not exist", caller
+      end
+
+      nested_paths << ::File.expand_path(path)
+    else
+      path.sub!(/#{File::SEPARATOR}$/, '')
+
+      # Make the path completely canonical
+      path = Pathname.new(path).expand_path
+
+      # Make sure the path is a valid directory
+      unless path.directory?
+        raise RuntimeError, "The path supplied is not a valid directory.", caller
+      end
+
+      nested_paths << path
+
+      # Identify any fastlib archives inside of this path
+      fastlib_glob = path.join('**', '*.fastlib')
+      Dir.glob(fastlib_glob).each do |fp|
+        nested_paths << fp
+      end
+    end
+
+    # Update the module paths appropriately
+    self.module_paths = (module_paths + nested_paths).flatten.uniq
+
+    # Load all of the modules from the nested paths
+    count_by_type = {}
+    nested_paths.each { |path|
+      path_count_by_type = load_modules(path, :force => false)
+
+      # merge hashes
+      path_count_by_type.each do |type, path_count|
+        accumulated_count = count_by_type.fetch(type, 0)
+        count_by_type[type] = accumulated_count + path_count
+      end
+    }
+
+    return count_by_type
+  end
+
+  #
+  # Removes a path from which to search for modules.
+  #
+  def remove_module_path(path)
+    module_paths.delete(path)
+    module_paths.delete(::File.expand_path(path))
+  end
+
+  protected
+
+  attr_accessor :module_paths # :nodoc:
+end

lib/msf/core/module_manager/module_sets.rb

@@ -0,0 +1,94 @@
+# Defines the MODULE_* constants
+require 'msf/core/constants'
+
+# Concerns the various type-specific module sets in a {Msf::ModuleManager}
+module Msf::ModuleManager::ModuleSets
+  extend ActiveSupport::Concern
+
+  #
+  # Returns the set of loaded auxiliary module classes.
+  #
+  def auxiliary
+    module_set(Msf::MODULE_AUX)
+  end
+
+  #
+  # Returns the set of loaded encoder module classes.
+  #
+  def encoders
+    module_set(Msf::MODULE_ENCODER)
+  end
+
+  #
+  # Returns the set of loaded exploit module classes.
+  #
+  def exploits
+    module_set(Msf::MODULE_EXPLOIT)
+  end
+
+  def init_module_set(type)
+    self.enablement_by_type[type] = true
+    case type
+      when Msf::MODULE_PAYLOAD
+        instance = Msf::PayloadSet.new(self)
+      else
+        instance = Msf::ModuleSet.new(type)
+    end
+
+    self.module_set_by_type[type] = instance
+
+    # Set the module set's framework reference
+    instance.framework = self.framework
+  end
+
+  #
+  # Provide a list of module names of a specific type
+  #
+  def module_names(set)
+    module_set_by_type[set] ? module_set_by_type[set].keys.dup : []
+  end
+
+  #
+  # Returns all of the modules of the specified type
+  #
+  def module_set(type)
+    module_set_by_type[type]
+  end
+
+  #
+  # Provide a list of the types of modules in the set
+  #
+  def module_types
+    module_set_by_type.keys.dup
+  end
+
+  #
+  # Returns the set of loaded nop module classes.
+  #
+  def nops
+    module_set(Msf::MODULE_NOP)
+  end
+
+  #
+  # Returns the set of loaded payload module classes.
+  #
+  def payloads
+    module_set(Msf::MODULE_PAYLOAD)
+  end
+
+  #
+  # Returns the set of loaded auxiliary module classes.
+  #
+  def post
+    module_set(Msf::MODULE_POST)
+  end
+
+  def type_enabled?(type)
+    enablement_by_type[type] || false
+  end
+
+  protected
+
+  attr_accessor :enablement_by_type # :nodoc:
+  attr_accessor :module_set_by_type # :nodoc:
+end

lib/msf/core/module_manager/reloading.rb

@@ -0,0 +1,49 @@
+# Concerns reloading modules
+module Msf::ModuleManager::Reloading
+  extend ActiveSupport::Concern
+
+  # Reloads the module specified in mod.  This can either be an instance of a module or a module class.
+  #
+  # @param [Msf::Module, Class] mod either an instance of a module or a module class
+  def reload_module(mod)
+    refname = mod.refname
+
+    dlog("Reloading module #{refname}...", 'core')
+
+    # if it's can instance, then get its class
+    if mod.is_a? Msf::Module
+      metasploit_class = mod.class
+    else
+      metasploit_class = mod
+    end
+
+    namespace_module = metasploit_class.parent
+    loader = namespace_module.loader
+    loader.reload_module(mod)
+  end
+
+  #
+  # Reloads modules from all module paths
+  #
+  def reload_modules
+    self.module_history = {}
+    self.clear
+
+    self.enablement_by_type.each_key do |type|
+      module_set_by_type[type].clear
+      init_module_set(type)
+    end
+
+    # The number of loaded modules in the following categories:
+    # auxiliary/encoder/exploit/nop/payload/post
+    count = 0
+    module_paths.each do |path|
+      mods = load_modules(path, true)
+      mods.each_value {|c| count += c}
+    end
+
+    rebuild_cache
+
+    count
+  end
+end

lib/msf/core/module_set.rb

@@ -0,0 +1,290 @@
+# -*- coding: binary -*-
+require 'msf/core'
+require 'fastlib'
+require 'pathname'
+
+module Msf
+
+  #
+  # Define used for a place-holder module that is used to indicate that the
+  # module has not yet been demand-loaded. Soon to go away.
+  #
+  SymbolicModule = "__SYMBOLIC__"
+
+  ###
+  #
+  # A module set contains zero or more named module classes of an arbitrary
+  # type.
+  #
+  ###
+  class ModuleSet < Hash
+    include Framework::Offspring
+
+    # Wrapper that detects if a symbolic module is in use.  If it is, it creates an instance to demand load the module
+    # and then returns the now-loaded class afterwords.
+    def [](name)
+      if (super == SymbolicModule)
+        create(name)
+      end
+
+      super
+    end
+
+    # Create an instance of the supplied module by its name
+    #
+    def create(name)
+      klass = fetch(name, nil)
+      instance = nil
+
+      # If there is no module associated with this class, then try to demand
+      # load it.
+      if klass.nil? or klass == SymbolicModule
+        # If we are the root module set, then we need to try each module
+        # type's demand loading until we find one that works for us.
+        if module_type.nil?
+          MODULE_TYPES.each { |type|
+            framework.modules.demand_load_module(type, name)
+          }
+        else
+          framework.modules.demand_load_module(module_type, name)
+        end
+
+        recalculate
+
+        klass = get_hash_val(name)
+      end
+
+      # If the klass is valid for this name, try to create it
+      if klass and klass != SymbolicModule
+        instance = klass.new
+      end
+
+      # Notify any general subscribers of the creation event
+      if instance
+        self.framework.events.on_module_created(instance)
+      end
+
+      return instance
+    end
+
+    #
+    # Overrides the builtin 'each' operator to avoid the following exception on Ruby 1.9.2+
+    #    "can't add a new key into hash during iteration"
+    #
+    def each(&block)
+      list = []
+      self.keys.sort.each do |sidx|
+        list << [sidx, self[sidx]]
+      end
+      list.each(&block)
+    end
+
+    #
+    # Enumerates each module class in the set.
+    #
+    def each_module(opts = {}, &block)
+      demand_load_modules
+
+      self.mod_sorted = self.sort
+
+      each_module_list(mod_sorted, opts, &block)
+    end
+
+    #
+    # Custom each_module filtering if an advanced set supports doing extended
+    # filtering.  Returns true if the entry should be filtered.
+    #
+    def each_module_filter(opts, name, entry)
+      return false
+    end
+
+    #
+    # Enumerates each module class in the set based on their relative ranking
+    # to one another.  Modules that are ranked higher are shown first.
+    #
+    def each_module_ranked(opts = {}, &block)
+      demand_load_modules
+
+      self.mod_ranked = rank_modules
+
+      each_module_list(mod_ranked, opts, &block)
+    end
+
+    #
+    # Forces all modules in this set to be loaded.
+    #
+    def force_load_set
+      each_module { |name, mod| }
+    end
+
+    #
+    # Initializes a module set that will contain modules of a specific type and
+    # expose the mechanism necessary to create instances of them.
+    #
+    def initialize(type = nil)
+      #
+      # Defaults
+      #
+      self.ambiguous_module_reference_name_set = Set.new
+      # Hashes that convey the supported architectures and platforms for a
+      # given module
+      self.mod_arch_hash     = {}
+      self.mod_platform_hash = {}
+      self.mod_sorted        = nil
+      self.mod_ranked        = nil
+      self.mod_extensions    = []
+
+      #
+      # Arguments
+      #
+      self.module_type = type
+    end
+
+    attr_reader   :module_type
+
+    #
+    # Gives the module set an opportunity to handle a module reload event
+    #
+    def on_module_reload(mod)
+    end
+
+    #
+    # Whether or not recalculations should be postponed.  This is used from the
+    # context of the each_module_list handler in order to prevent the demand
+    # loader from calling recalc for each module if it's possible that more
+    # than one module may be loaded.  This field is not initialized until used.
+    #
+    attr_accessor :postpone_recalc
+
+
+    #
+    # Dummy placeholder to relcalculate aliases and other fun things.
+    #
+    def recalculate
+    end
+
+    #
+    # Checks to see if the supplied module name is valid.
+    #
+    def valid?(name)
+      create(name)
+      (self[name]) ? true : false
+    end
+
+    protected
+
+    #
+    # Adds a module with a the supplied name.
+    #
+    def add_module(mod, name, modinfo = nil)
+      # Set the module's name so that it can be referenced when
+      # instances are created.
+      mod.framework = framework
+      mod.refname   = name
+      mod.file_path = ((modinfo and modinfo['files']) ? modinfo['files'][0] : nil)
+      mod.orig_cls  = mod
+
+      cached_module = self[name]
+
+      if (cached_module and cached_module != SymbolicModule)
+        ambiguous_module_reference_name_set.add(name)
+
+        wlog("The module #{mod.refname} is ambiguous with #{self[name].refname}.")
+      else
+        self[name] = mod
+      end
+
+      mod
+    end
+
+    #
+    # Load all modules that are marked as being symbolic.
+    #
+    def demand_load_modules
+      # Pre-scan the module list for any symbolic modules
+      self.each_pair { |name, mod|
+        if (mod == SymbolicModule)
+          self.postpone_recalc = true
+
+          mod = create(name)
+
+          next if (mod.nil?)
+        end
+      }
+
+      # If we found any symbolic modules, then recalculate.
+      if (self.postpone_recalc)
+        self.postpone_recalc = false
+
+        recalculate
+      end
+    end
+
+    #
+    # Enumerates the modules in the supplied array with possible limiting
+    # factors.
+    #
+    def each_module_list(ary, opts, &block)
+      ary.each { |entry|
+        name, mod = entry
+
+        # Skip any lingering symbolic modules.
+        next if (mod == SymbolicModule)
+
+        # Filter out incompatible architectures
+        if (opts['Arch'])
+          if (!mod_arch_hash[mod])
+            mod_arch_hash[mod] = mod.new.arch
+          end
+
+          next if ((mod_arch_hash[mod] & opts['Arch']).empty? == true)
+        end
+
+        # Filter out incompatible platforms
+        if (opts['Platform'])
+          if (!mod_platform_hash[mod])
+            mod_platform_hash[mod] = mod.new.platform
+          end
+
+          next if ((mod_platform_hash[mod] & opts['Platform']).empty? == true)
+        end
+
+        # Custom filtering
+        next if (each_module_filter(opts, name, entry) == true)
+
+        block.call(name, mod)
+      }
+    end
+
+    attr_accessor :ambiguous_module_reference_name_set
+    attr_accessor :mod_arch_hash
+    attr_accessor :mod_extensions
+    attr_accessor :mod_platform_hash
+    attr_accessor :mod_ranked
+    attr_accessor :mod_sorted
+    attr_writer   :module_type
+    attr_accessor :module_history
+
+    #
+    # Ranks modules based on their constant rank value, if they have one.
+    #
+    def rank_modules
+      self.mod_ranked = self.sort { |a, b|
+        a_name, a_mod = a
+        b_name, b_mod = b
+
+        # Dynamically loads the module if needed
+        a_mod = create(a_name) if a_mod == SymbolicModule
+        b_mod = create(b_name) if b_mod == SymbolicModule
+
+        # Extract the ranking between the two modules
+        a_rank = a_mod.const_defined?('Rank') ? a_mod.const_get('Rank') : NormalRanking
+        b_rank = b_mod.const_defined?('Rank') ? b_mod.const_get('Rank') : NormalRanking
+
+        # Compare their relevant rankings.  Since we want highest to lowest,
+        # we compare b_rank to a_rank in terms of higher/lower precedence
+        b_rank <=> a_rank
+      }
+    end
+  end
+end

lib/msf/core/modules.rb

@@ -0,0 +1,4 @@
+# Namespace for loading Metasploit modules
+module Msf::Modules
+
+end

lib/msf/core/modules/loader.rb

@@ -0,0 +1,6 @@
+require 'msf/core/modules'
+
+# Namespace for module loaders
+module Msf::Modules::Loader
+
+end

lib/msf/core/modules/loader/archive.rb

@@ -0,0 +1,77 @@
+require 'msf/core/modules/loader/base'
+
+# Concerns loading modules form fastlib archives
+class Msf::Modules::Loader::Archive < Msf::Modules::Loader::Base
+  #
+  # CONSTANTS
+  #
+
+  # The extension for Fastlib archives.
+  ARCHIVE_EXTENSION = '.fastlib'
+
+  # Returns true if the path is a Fastlib archive.
+  #
+  # @param (see Msf::Modules::Loader::Base#loadable?)
+  # @return [true] if path has the {ARCHIVE_EXTENSION} extname.
+  # @return [false] otherwise
+  def loadable?(path)
+    if File.extname(path) == ARCHIVE_EXTENSION
+      true
+    else
+      false
+    end
+  end
+
+  protected
+
+  # Yields the module_reference_name for each module file in the Fastlib archive at path.
+  #
+  # @param path [String] The path to the Fastlib archive file.
+  # @yield (see Msf::Modules::Loader::Base#each_module_reference_name)
+  # @yieldparam (see Msf::Modules::Loader::Base#each_module_reference_name)
+  # @return (see Msf::Modules::Loader::Base#each_module_reference_name)
+  def each_module_reference_name(path)
+    entries = ::FastLib.list(path)
+
+    entries.each do |entry|
+      if entry.include?('.svn/')
+        next
+      end
+
+      type = entry.split('/', 2)[0]
+      type = type.singularize
+
+      unless module_manager.enablement_by_type[type]
+        next
+      end
+
+      if module_path?(entry)
+        # The module_reference_name doesn't have a file extension
+        module_reference_name = module_reference_name_from_path(entry)
+
+        yield path, type, module_reference_name
+      end
+    end
+  end
+
+  # Returns the path to the module inside the Fastlib archive.  The path to the archive is separated from the path to
+  # the file inside the archive by '::'.
+  #
+  # @param (see Msf::Modules::Loader::Base#module_path)
+  # @return [String] Path to module file inside the Fastlib archive.
+  def module_path(parent_path, type, module_reference_name)
+    file_path = typed_path(type, module_reference_name)
+    module_path = "#{parent_path}::#{file_path}"
+
+    module_path
+  end
+
+  # Loads the module content from the Fastlib archive.
+  #
+  # @return (see Msf::Modules::Loader::Base#read_module_content)
+  def read_module_content(path, type, module_reference_name)
+    file_path = typed_path(type, module_reference_name)
+
+    ::FastLib.load(path, file_path)
+  end
+end

lib/msf/core/modules/loader/base.rb

@@ -0,0 +1,477 @@
+#
+# Project
+#
+require 'msf/core/modules/loader'
+require 'msf/core/modules/namespace'
+
+# Responsible for loading modules for {Msf::ModuleManagers}.
+#
+# @abstract Subclass and override {#base_path}, {#each_module_reference_name}, {#loadable?}, and
+#   {#read_module_content}.
+class Msf::Modules::Loader::Base
+  #
+  # CONSTANTS
+  #
+
+  # This must calculate the first line of the NAMESPACE_MODULE_CONTENT string so that errors are reported correctly
+  NAMESPACE_MODULE_LINE = __LINE__ + 4
+  # By calling module_eval from inside the module definition, the lexical scope is captured and available to the code in
+  # module_content.
+  NAMESPACE_MODULE_CONTENT = <<-EOS
+    # ensure the namespace module can respond to checks during loading
+    extend Msf::Modules::Namespace
+
+    class << self
+      # The loader that originally loaded this module
+      #
+      # @return [Msf::Modules::Loader::Base] the loader that loaded this namespace module and can reload it.
+      attr_accessor :loader
+
+      # @return [String] The path under which the module of the given type was found.
+      attr_accessor :parent_path
+    end
+
+    # Calls module_eval on the module_content, but the lexical scope of the namespace_module is passed through
+    # module_eval, so that module_content can act like it was written inline in the namespace_module.
+    #
+    # @param [String] module_content The content of the {Msf::Module}.
+    # @param [String] module_path The path to the module, so that error messages in evaluating the module_content can
+    #   be reported correctly.
+    def self.module_eval_with_lexical_scope(module_content, module_path)
+      # By calling module_eval from inside the module definition, the lexical scope is captured and available to the
+      # code in module_content.
+      module_eval(module_content)
+    end
+  EOS
+
+  # The extension for metasploit modules.
+  MODULE_EXTENSION = '.rb'
+  # String used to separate module names in a qualified module name.
+  MODULE_SEPARATOR = '::'
+  # The base namespace name under which {#namespace_module #namespace_modules} are created.
+  NAMESPACE_MODULE_NAMES = ['Msf', 'Modules']
+  # Regex that can distinguish regular ruby source from unit test source.
+  UNIT_TEST_REGEX = /rb\.(ut|ts)\.rb$/
+  # Not all types are pluralized when a directory name, so here's the mapping that currently exists
+  DIRECTORY_BY_TYPE = {
+    Msf::MODULE_AUX => 'auxiliary',
+    Msf::MODULE_ENCODER => 'encoders',
+    Msf::MODULE_EXPLOIT => 'exploits',
+    Msf::MODULE_NOP => 'nops',
+    Msf::MODULE_PAYLOAD => 'payloads',
+    Msf::MODULE_POST => 'post'
+  }
+
+  # @param [Msf::ModuleManager] module_manager The module manager that caches the loaded modules.
+  def initialize(module_manager)
+    @module_manager = module_manager
+  end
+
+  # Returns whether the path can be loaded this module loader.
+  #
+  # @param path (see #load_modules)
+  # @return [Boolean]
+  def loadable?(path)
+    raise ::NotImplementedError
+  end
+
+  # Loads all of the modules from the supplied path.
+  #
+  # @note Only paths where {#loadable?} returns true should be passed to this method.
+  #
+  # @param [String] path Path under which there are modules
+  # @param [Hash] options
+  # @option options [Boolean] force (false) whether to force loading of the module even if the module has not changed.
+  # @return [Hash{String => Integer}] Maps module type to number of modules loaded
+  def load_modules(path, options={})
+    options.assert_valid_keys(:force)
+
+    force = options[:force]
+    count_by_type = {}
+    recalculate_by_type = {}
+
+    each_module_reference_name(path) do |parent_path, type, module_reference_name|
+      load_module(
+          parent_path,
+          type,
+          module_reference_name,
+          :recalculate_by_type => recalculate_by_type,
+          :count_by_type => count_by_type,
+          :force => force
+      )
+    end
+
+    recalculate_by_type.each do |type, recalculate|
+      if recalculate
+        module_set = module_manager.module_set(type)
+        module_set.recalculate
+      end
+    end
+
+    count_by_type
+  end
+
+  # Reloads the specified module.
+  #
+  # @param [Class, Msf::Module] original_metasploit_class_or_instance either an instance of a module or a module class
+  def reload_module(original_metasploit_class_or_instance)
+    if original_metasploit_class_or_instance.is_a? Msf::Module
+      original_metasploit_instance = original_metasploit_class_or_instance
+      original_metasploit_class = original_metasploit_class_or_instance.class
+    else
+      original_metasploit_instance = nil
+      original_metasploit_class = original_metasploit_class_or_instance
+    end
+
+    namespace_module = original_metasploit_class.parent
+    parent_path = namespace_module.parent_path
+
+    type = original_metasploit_class_or_instance.type
+    module_reference_name = original_metasploit_class_or_instance.refname
+
+    dlog("Reloading module #{module_reference_name}...", 'core')
+
+    if load_module(parent_path, type, module_reference_name, :force => true)
+      # Create a new instance of the module
+      reloaded_module_instance = module_manager.create(module_reference_name)
+
+      if reloaded_module_instance and original_metasploit_instance
+        # copy over datastore
+        reloaded_module_instance.datastore.update(original_metasploit_instance.datastore)
+      else
+        elog("Failed to create instance of #{refname} after reload.", 'core')
+
+        # Return the old module instance to avoid a strace trace
+        return original_module
+      end
+    else
+      elog("Failed to reload #{module_reference_name}")
+
+      # Return the old module isntance to avoid a strace trace
+      return original_metasploit_class_or_instance
+    end
+
+    # Let the specific module sets have an opportunity to handle the fact
+    # that this module was reloaded.
+    module_set = module_manager.module_set(type)
+    module_set.on_module_reload(reloaded_module_instance)
+
+    # Rebuild the cache for just this module
+    module_manager.rebuild_cache(reloaded_module_instance)
+
+    reloaded_module_instance
+  end
+
+  protected
+
+  # Yields module reference names under path.
+  #
+  # @param path (see #load_modules)
+  # @yield [parent_path, type, module_reference_name] Gives the path and the module_reference_name of the module found
+  #   under the path.
+  # @yieldparam parent_path [String] the path under which the module of the given type was found.
+  # @yieldparam type [String] the type of the module.
+  # @yieldparam module_reference_name [String] The canonical name for referencing the module.
+  # @return [void]
+  def each_module_reference_name(path)
+    raise ::NotImplementedError
+  end
+
+  # Loads a module from the supplied path and module_reference_name.
+  #
+  # @param [String] parent_path The path under which the module exists.  This is not necessarily the same path as passed
+  #   to {#load_modules}: it may just be derived from that path.
+  # @param [String] type The type of module.
+  # @param [String] module_reference_name The canonical name for referring to the module.
+  # @param [Hash] options Options used to force loading and track statistics
+  # @option options [Hash{String => Integer}] :count_by_type Maps the module type to the number of module loaded
+  # @option options [Boolean] :force (false) whether to force loading of the module even if the module has not changed.
+  # @option options [Hash{String => Boolean}] :recalculate_by_type Maps type to whether its
+  #   {Msf::ModuleManager#module_set} needs to be recalculated.
+  def load_module(parent_path, type, module_reference_name, options={})
+    options.assert_valid_keys(:count_by_type, :force, :recalculate_by_type)
+    force = options[:force] || false
+
+    unless force or module_manager.file_changed?(parent_path)
+      dlog("Cached module from #{parent_path} has not changed.", 'core', LEV_2)
+
+      return false
+    end
+
+    namespace_module = self.namespace_module(module_reference_name)
+
+    # set the parent_path so that the module can be reloaded with #load_module
+    namespace_module.parent_path = parent_path
+
+    module_content = read_module_content(parent_path, type, module_reference_name)
+    module_path = module_path(parent_path, type, module_reference_name)
+
+    begin
+      namespace_module.module_eval_with_lexical_scope(module_content, module_path)
+    # handle interrupts as pass-throughs unlike other Exceptions
+    rescue ::Interrupt
+      raise
+    rescue ::Exception => error
+      # Hide eval errors when the module version is not compatible
+      begin
+        namespace_module.version_compatible!
+      rescue Msf::Modules::VersionCompatibilityError => version_compatibility_error
+        error_message = "Failed to load module (#{module_path}) due to error and #{version_compatibility_error}"
+      else
+        error_message = "#{error.class} #{error}:\n#{error.backtrace}"
+      end
+
+      elog(error_message)
+      module_manager.module_load_error_by_reference_name[module_reference_name] = error_message
+
+      return false
+    end
+
+    metasploit_class = namespace_module.metasploit_class
+
+    unless metasploit_class
+      error_message = "Missing Metasploit class constant"
+
+      elog(error_message)
+      module_manager.module_load_error_by_reference_name[module_reference_name] = error_message
+    end
+
+    unless usable?(metasploit_class)
+      ilog("Skipping module #{module_reference_name} under #{parent_path} because is_usable returned false.", 'core', LEV_1)
+
+      return false
+    end
+
+    ilog("Loaded #{type} module #{module_reference_name} under #{parent_path}", 'core', LEV_2)
+
+    # if this is a reload, then there may be a pre-existing error that should now be cleared
+    module_manager.module_load_error_by_reference_name.delete(module_reference_name)
+
+    # Do some processing on the loaded module to get it into the right associations
+    module_manager.on_module_load(
+        metasploit_class,
+        type,
+        module_reference_name,
+        {
+            # files[0] is stored in the {Msf::Module#file_path} and is used to reload the module, so it needs to be a
+            # full path
+            'files' => [
+                module_path
+            ],
+            'paths' => [
+                module_reference_name
+            ],
+            'type' => type
+        }
+    )
+
+    # Set this module type as needing recalculation
+    recalculate_by_type = options[:recalculate_by_type]
+
+    if recalculate_by_type
+      recalculate_by_type[type] = true
+    end
+
+    # The number of loaded modules this round
+    count_by_type = options[:count_by_type]
+
+    if count_by_type
+      count_by_type[type] ||= 0
+      count_by_type[type] += 1
+    end
+
+    return true
+  end
+
+  attr_reader :module_manager
+
+  # Returns path to module that can be used for reporting errors in evaluating the
+  # {#read_module_content module_content}.
+  #
+  # @param path (see #load_module)
+  # @param type (see #load_module)
+  # @param module_reference_name (see #load_module)
+  # @return [String] The path to module.
+  def module_path(parent_path, type, module_reference_name)
+    raise ::NotImplementedError
+  end
+
+  # Returns whether the path could refer to a module.  The path would still need to be loaded in order to check if it
+  # actually is a valid module.
+  #
+  # @param [String] path to module without the type directory.
+  # @return [true] if the extname is {MODULE_EXTENSION} AND
+  #                   the path does not match {UNIT_TEXT_REGEX} AND
+  #                   the path is not hidden (starts with '.')
+  # @return [false] otherwise
+  def module_path?(path)
+    module_path = false
+
+    extension = File.extname(path)
+
+    unless (path.starts_with?('.') or
+            extension != MODULE_EXTENSION or
+            path =~ UNIT_TEST_REGEX)
+      module_path = true
+    end
+
+    module_path
+  end
+
+  # Changes a file name path to a canonical module reference name.
+  #
+  # @param [String] path Relative path to module.
+  # @return [String] MODULE_EXTENSION removed from path.
+  def module_reference_name_from_path(path)
+    path.gsub(/#{MODULE_EXTENSION}$/, '')
+  end
+
+  # Returns a nested module to wrap the Metasploit(1|2|3) class so that it doesn't overwrite other (metasploit)
+  # module's classes.  The wrapper module must be named so that active_support's autoloading code doesn't break when
+  # searching constants from inside the Metasploit(1|2|3) class.
+  def namespace_module(module_reference_name)
+    namespace_module_names = self.namespace_module_names(module_reference_name)
+
+    # If this is a reload, then the pre-existing namespace_module needs to be removed.
+
+    # don't check ancestors for the constants
+    inherit = false
+
+    # Don't want to trigger ActiveSupport's const_missing, so can't use constantize.
+    namespace_module = namespace_module_names.inject(Object) { |parent, module_name|
+      if parent.const_defined?(module_name, inherit)
+        parent.const_get(module_name, inherit)
+      else
+        break
+      end
+    }
+
+    # if a reload
+    unless namespace_module.nil?
+      parent_module = namespace_module.parent
+
+      # remove_const is private, so invoke in instance context
+      parent_module.instance_eval do
+        remove_const namespace_module_names.last
+      end
+    end
+
+    # In order to have constants defined in Msf resolve without the Msf qualifier in the module_content, the
+    # Module.nesting must resolve for the entire nesting.  Module.nesting is strictly lexical, and can't be faked with
+    # module_eval(&block). (There's actually code in ruby's implementation to stop module_eval from being added to
+    # Module.nesting when using the block syntax.) All this means is the modules have to be declared as a string that
+    # gets module_eval'd.
+
+    nested_module_names = namespace_module_names.reverse
+
+    namespace_module_content = nested_module_names.inject(NAMESPACE_MODULE_CONTENT) { |wrapped_content, module_name|
+      lines = []
+      lines << "module #{module_name}"
+      lines << wrapped_content
+      lines << "end"
+
+      lines.join("\n")
+    }
+
+    Object.module_eval(namespace_module_content, __FILE__, NAMESPACE_MODULE_LINE)
+
+    # The namespace_module exists now, so no need to use constantize to do const_missing
+    namespace_module = namespace_module_names.inject(Object) { |parent, module_name|
+      parent.const_get(module_name, inherit)
+    }
+    # record the loader, so that the namespace module and its metasploit_class can be reloaded
+    namespace_module.loader = self
+
+    namespace_module
+  end
+
+  def namespace_module_name(module_reference_name)
+    namespace_module_names = self.namespace_module_names(module_reference_name)
+    namespace_module_name = namespace_module_names.join(MODULE_SEPARATOR)
+
+    namespace_module_name
+  end
+
+  # Returns a fully qualified module name to wrap the Metasploit(1|2|3) class so that it doesn't overwrite other
+  # (metasploit) module's classes.  Invalid module name characters are escaped by using 'H*' unpacking and prefixing
+  # each code with X so the code remains a valid module name when it starts with a digit.
+  #
+  # @param [String] module_reference_name The canonical name for the module.
+  # @return [Module] Msf::Modules::<name>
+  #
+  # @see namespace_module
+  def namespace_module_names(module_reference_name)
+    relative_module_name = module_reference_name.camelize
+
+    module_names = relative_module_name.split('::')
+
+    # The module_reference_name is path-like, so it can include characters that are invalid in module names
+    valid_module_names = module_names.collect { |module_name|
+      valid_module_name = module_name.gsub(/^[0-9]|[^A-Za-z0-9]/) { |invalid_constant_name_character|
+        unpacked = invalid_constant_name_character.unpack('H*')
+        # unpack always returns an array, so get first value to get character's encoding
+        hex_code = unpacked[0]
+
+        # as a convention start each hex-code with X so that it'll make a valid constant name since constants can't
+        # start with digits.
+        "X#{hex_code}"
+      }
+
+      valid_module_name
+    }
+
+    namespace_module_names = NAMESPACE_MODULE_NAMES + valid_module_names
+
+    namespace_module_names
+  end
+
+  # Read the content of the module from under path.
+  #
+  # @param parent_path (see #load_module)
+  # @param type (see #load_module)
+  # @param module_reference_name (see #load_module)
+  # @return [String] module content that can be module_evaled into the {#namespace_module}
+  def read_module_content(parent_path, type, module_reference_name)
+    raise ::NotImplementedError
+  end
+
+
+  # The path to the module qualified by the type directory.
+  #
+  # @note To get the full path to the module, use {#module_path}
+  #
+  # @param [String] type The type of the module.
+  # @param [String] module_reference_name The canonical name for the module.
+  # @return [String] path to the module starting with the type directory.
+  #
+  # @see DIRECTORY_BY_TYPE
+  def typed_path(type, module_reference_name)
+    file_name = module_reference_name + MODULE_EXTENSION
+    type_directory = DIRECTORY_BY_TYPE[type]
+    typed_path = File.join(type_directory, file_name)
+
+    typed_path
+  end
+
+  # Returns whether the metasploit_class is usable on the current system.  Defer's to metasploit_class's #is_usable if
+  # it is defined.
+  #
+  # @param [Msf::Module] metasploit_class As returned by {Msf::Modules::Namespace#metasploit_class}
+  def usable?(metasploit_class)
+    # If the module indicates that it is not usable on this system, then we
+    # will not try to use it.
+    usable = false
+
+    if metasploit_class.respond_to? :is_usable
+      begin
+        usable = metasploit_class.is_usable
+      rescue => error
+        elog("Exception caught during is_usable check: #{error}")
+      end
+    else
+      usable = true
+    end
+
+    usable
+  end
+end

lib/msf/core/modules/loader/directory.rb

@@ -0,0 +1,79 @@
+# Concerns loading module from a directory
+class Msf::Modules::Loader::Directory < Msf::Modules::Loader::Base
+  # Returns true if the path is a directory
+  #
+  # @param (see Msf::Modules::Loader::Base#loadable?)
+  # @return [true] if path is a directory
+  # @return [false] otherwise
+  def loadable?(path)
+    if File.directory?(path)
+      true
+    else
+      false
+    end
+  end
+
+  protected
+
+  # Yields the module_reference_name for each module file found under the directory path.
+  #
+  # @param [String] path The path to the directory.
+  # @yield (see Msf::Modules::Loader::Base#each_module_reference_name)
+  # @yieldparam [String] path The path to the directory.
+  # @yieldparam [String] type The type correlated with the directory under path.
+  # @yieldparam module_reference_name (see Msf::Modules::Loader::Base#each_module_reference_name)
+  # @return (see Msf::Modules::Loader::Base#each_module_reference_name)
+  def each_module_reference_name(path)
+    ::Dir.foreach(path) do |entry|
+      if entry.downcase == '.svn'
+        next
+      end
+
+      full_entry_path = ::File.join(path, entry)
+      type = entry.singularize
+
+      unless ::File.directory?(full_entry_path) and
+             module_manager.type_enabled? type
+        next
+      end
+
+      full_entry_pathname = Pathname.new(full_entry_path)
+
+      # Try to load modules from all the files in the supplied path
+      Rex::Find.find(full_entry_path) do |entry_descendant_path|
+        if module_path?(entry_descendant_path)
+          entry_descendant_pathname = Pathname.new(entry_descendant_path)
+          relative_entry_descendant_pathname = entry_descendant_pathname.relative_path_from(full_entry_pathname)
+          relative_entry_descendant_path = relative_entry_descendant_pathname.to_path
+
+          # The module_reference_name doesn't have a file extension
+          module_reference_name = module_reference_name_from_path(relative_entry_descendant_path)
+
+          yield path, type, module_reference_name
+        end
+      end
+    end
+  end
+
+  # Returns the full path to the module file on disk.
+  #
+  # @param (see Msf::Modules::Loader::Base#module_path)
+  # @return [String] Path to module file on disk.
+  def module_path(parent_path, type, module_reference_name)
+    file_name = module_reference_name + MODULE_EXTENSION
+    type_directory = DIRECTORY_BY_TYPE[type]
+    full_path = File.join(parent_path, type_directory, file_name)
+
+    full_path
+  end
+
+  # Loads the module content from the on disk file.
+  #
+  # @param (see Msf::Modules::Loader::Base#read_module_content)
+  # @return (see Msf::Modules::Loader::Base#read_module_content)
+  def read_module_content(parent_path, type, module_reference_name)
+    full_path = module_path(parent_path, type, module_reference_name)
+
+    ::File.read(full_path)
+  end
+end

lib/msf/core/modules/namespace.rb

@@ -0,0 +1,51 @@
+# Concern for behavior that all namespace modules that wrap Msf::Modules must support like version checking and
+# grabbing the version specific-Metasploit* class.
+module Msf::Modules::Namespace
+  # Returns the Metasploit(3|2|1) class from the module_evalled content.
+  #
+  # @note The module content must be module_evalled into this namespace module before the return of
+  #   {metasploit_class} is valid.
+  #
+  # @return [Msf::Module] if a Metasploit(3|2|1) class exists in this module
+  # @return [nil] if such as class is not defined.
+  def metasploit_class
+    metasploit_class = nil
+    # don't search ancestors for the metasploit_class
+    inherit = false
+
+    ::Msf::Framework::Major.downto(1) do |major|
+      if const_defined?("Metasploit#{major}", inherit)
+        metasploit_class = const_get("Metasploit#{major}")
+
+        break
+      end
+    end
+
+    metasploit_class
+  end
+
+  # Raises an error unless {Msf::Framework::VersionCore} and {Msf::Framework::VersionAPI} meet the minimum required
+  # versions defined in RequiredVersions in the module content.
+  #
+  # @note The module content must be module_evalled into this namespace module using module_eval_with_lexical_scope
+  #   before calling {version_compatible!} is valid.
+  #
+  # @raise [Msf::Modules::VersionCompatibilityError] if RequiredVersion[0] > Msf::Framework::VersionCore or
+  #   RequiredVersion[1] > Msf::Framework::VersionApi
+  # @return [void]
+  def version_compatible!
+    if const_defined?(:RequiredVersions)
+      required_versions = const_get(:RequiredVersions)
+      minimum_core_version = required_versions[0]
+      minimum_api_version = required_versions[1]
+
+      if (minimum_core_version > ::Msf::Framework::VersionCore or
+          minimum_api_version > ::Msf::Framework::VersionAPI)
+        raise Msf::Modules::VersionCompatibilityError.new(
+                  :minimum_api_version => minimum_api_version,
+                  :minimum_core_version => minimum_core_version
+              )
+      end
+    end
+  end
+end

lib/msf/core/modules/version_compatibility_error.rb

@@ -0,0 +1,12 @@
+class Msf::Modules::VersionCompatibilityError < StandardError
+  def initialize(attributes={})
+    @minimum_api_version = attributes[:minimum_api_version]
+    @minimum_core_version = attributes[:minimum_core_version]
+
+    super("Failed to reload module (#{name}) due to version check " \
+          "(requires API:#{minimum_api_version} Core:#{minimum_core_version})")
+  end
+
+  attr_reader :minimum_api_version
+  attr_reader :minimum_core_version
+end