infosecn1nja
/
metasploit-framework
mirror of https://github.com/infosecn1nja/metasploit-framework.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'landing/4189' into upstream-master 

Land #4189

* Detect leaked threads during spec runs
* Manage threads before/after spec runs
bug/bundler_fix
Trevor Rosen 2014-11-18 08:33:38 -06:00
parent 7daedac399 e3869ee1ae
commit fff36f5968
No known key found for this signature in database
GPG Key ID: 255ADB7A642D3928
12 changed files with 1084 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Rakefile
View File

@ -11,4 +11,5 @@ Metasploit::Framework::Require.optionally_active_record_railtie
Metasploit::Framework::Application.load_tasks
Metasploit::Framework::Spec::Constants.define_task
Metasploit::Framework::Spec::Threads::Suite.define_task
Metasploit::Framework::Spec::UntestedPayloads.define_task

1
lib/metasploit/framework/spec.rb
View File

@ -2,4 +2,5 @@ module Metasploit::Framework::Spec
extend ActiveSupport::Autoload
autoload :Constants
autoload :Threads
end

5
lib/metasploit/framework/spec/threads.rb Normal file
View File

@ -0,0 +1,5 @@
module Metasploit::Framework::Spec::Threads
extend ActiveSupport::Autoload
autoload :Suite
end

41
lib/metasploit/framework/spec/threads/logger.rb Normal file
View File

@ -0,0 +1,41 @@
#
# Standard Library
#
require 'securerandom'
#
# Project
#
require 'metasploit/framework/spec/threads/suite'
original_thread_new = Thread.method(:new)
# Patches `Thread.new` so that if logs `caller` so thread leaks can be traced
Thread.define_singleton_method(:new) { |*args, &block|
uuid = SecureRandom.uuid
# tag caller with uuid so that only leaked threads caller needs to be printed
lines = ["BEGIN Thread.new caller (#{uuid})"]
caller.each do |frame|
lines << " #{frame}"
end
lines << 'END Thread.new caller'
Metasploit::Framework::Spec::Threads::Suite::LOG_PATHNAME.parent.mkpath
Metasploit::Framework::Spec::Threads::Suite::LOG_PATHNAME.open('a') { |f|
# single puts so threads can't write in between each other.
f.puts lines.join("\n")
}
options = {original_args: args, uuid: uuid}
original_thread_new.call(options) {
# record uuid for thread-leak detection can used uuid to correlate log with this thread.
Thread.current[Metasploit::Framework::Spec::Threads::Suite::UUID_THREAD_LOCAL_VARIABLE] = options.fetch(:uuid)
block.call(*options.fetch(:original_args))
}
}

214
lib/metasploit/framework/spec/threads/suite.rb Normal file
View File

@ -0,0 +1,214 @@
require 'pathname'
# @note needs to use explicit nesting. so this file can be loaded directly without loading 'metasploit/framework' which
# allows for faster loading of rake tasks.
module Metasploit
module Framework
module Spec
module Threads
module Suite
#
# CONSTANTS
#
# Number of allowed threads when threads are counted in `after(:suite)` or `before(:suite)`
EXPECTED_THREAD_COUNT_AROUND_SUITE = 1
# `caller` for all Thread.new calls
LOG_PATHNAME = Pathname.new('log/metasploit/framework/spec/threads/suite.log')
# Regular expression for extracting the UUID out of {LOG_PATHNAME} for each Thread.new caller block
UUID_REGEXP = /BEGIN Thread.new caller \((?<uuid>.*)\)/
# Name of thread local variable that Thread UUID is stored
UUID_THREAD_LOCAL_VARIABLE = "metasploit/framework/spec/threads/logger/uuid"
#
# Module Methods
#
# Configures `before(:suite)` and `after(:suite)` callback to detect thread leaks.
#
# @return [void]
def self.configure!
unless @configured
RSpec.configure do |config|
config.before(:suite) do
thread_count = Metasploit::Framework::Spec::Threads::Suite.non_debugger_thread_list.count
# check with if first so that error message can be constructed lazily
if thread_count > EXPECTED_THREAD_COUNT_AROUND_SUITE
# LOG_PATHNAME may not exist if suite run without `rake spec`
if LOG_PATHNAME.exist?
log = LOG_PATHNAME.read()
else
log "Run `rake spec` to log where Thread.new is called."
end
raise RuntimeError,
"#{thread_count} #{'thread'.pluralize(thread_count)} exist(s) when " \
"only #{EXPECTED_THREAD_COUNT_AROUND_SUITE} " \
"#{'thread'.pluralize(EXPECTED_THREAD_COUNT_AROUND_SUITE)} expected before suite runs:\n" \
"#{log}"
end
LOG_PATHNAME.parent.mkpath
LOG_PATHNAME.open('a') do |f|
# separator so after(:suite) can differentiate between threads created before(:suite) and during the
# suites
f.puts 'before(:suite)'
end
end
config.after(:suite) do
LOG_PATHNAME.parent.mkpath
LOG_PATHNAME.open('a') do |f|
# separator so that a flip flop can be used when reading the file below. Also useful if it turns
# out any threads are being created after this callback, which could be the case if another
# after(:suite) accidentally created threads by creating an Msf::Simple::Framework instance.
f.puts 'after(:suite)'
end
thread_list = Metasploit::Framework::Spec::Threads::Suite.non_debugger_thread_list
thread_count = thread_list.count
if thread_count > EXPECTED_THREAD_COUNT_AROUND_SUITE
error_lines = []
if LOG_PATHNAME.exist?
caller_by_thread_uuid = Metasploit::Framework::Spec::Threads::Suite.caller_by_thread_uuid
thread_list.each do |thread|
thread_uuid = thread[Metasploit::Framework::Spec::Threads::Suite::UUID_THREAD_LOCAL_VARIABLE]
# unmanaged thread, such as the main VM thread
unless thread_uuid
next
end
caller = caller_by_thread_uuid[thread_uuid]
error_lines << "Thread #{thread_uuid}'s status is #{thread.status.inspect} " \
"and was started here:\n"
error_lines.concat(caller)
end
else
error_lines << "Run `rake spec` to log where Thread.new is called."
end
raise RuntimeError,
"#{thread_count} #{'thread'.pluralize(thread_count)} exist(s) when only " \
"#{EXPECTED_THREAD_COUNT_AROUND_SUITE} " \
"#{'thread'.pluralize(EXPECTED_THREAD_COUNT_AROUND_SUITE)} expected after suite runs:\n" \
"#{error_lines.join}"
end
end
end
@configured = true
end
@configured
end
def self.define_task
Rake::Task.define_task('metasploit:framework:spec:threads:suite') do
if Metasploit::Framework::Spec::Threads::Suite::LOG_PATHNAME.exist?
Metasploit::Framework::Spec::Threads::Suite::LOG_PATHNAME.delete
end
parent_pathname = Pathname.new(__FILE__).parent
threads_logger_pathname = parent_pathname.join('logger')
load_pathname = parent_pathname.parent.parent.parent.parent.expand_path
# Must append to RUBYOPT or Rubymine debugger will not work
ENV['RUBYOPT'] = "#{ENV['RUBYOPT']} -I#{load_pathname} -r#{threads_logger_pathname}"
end
Rake::Task.define_task(spec: 'metasploit:framework:spec:threads:suite')
end
# @note Ensure {LOG_PATHNAME} exists before calling.
#
# Yields each line of {LOG_PATHNAME} that happened during the suite run.
#
# @yield [line]
# @yieldparam line [String] a line in the {LOG_PATHNAME} between `before(:suite)` and `after(:suite)`
# @yieldreturn [void]
def self.each_suite_line
in_suite = false
LOG_PATHNAME.each_line do |line|
if in_suite
if line.start_with?('after(:suite)')
break
else
yield line
end
else
if line.start_with?('before(:suite)')
in_suite = true
end
end
end
end
# @note Ensure {LOG_PATHNAME} exists before calling.
#
# Yield each line for each Thread UUID gathered during the suite run.
#
# @yield [uuid, line]
# @yieldparam uuid [String] the UUID of thread thread
# @yieldparam line [String] a line in the `caller` for the given `uuid`
# @yieldreturn [void]
def self.each_thread_line
in_thread_caller = false
uuid = nil
each_suite_line do |line|
if in_thread_caller
if line.start_with?('END Thread.new caller')
in_thread_caller = false
next
else
yield uuid, line
end
else
match = line.match(UUID_REGEXP)
if match
in_thread_caller = true
uuid = match[:uuid]
end
end
end
end
# The `caller` for each Thread UUID.
#
# @return [Hash{String => Array<String>}]
def self.caller_by_thread_uuid
lines_by_thread_uuid = Hash.new { |hash, uuid|
hash[uuid] = []
}
each_thread_line do |uuid, line|
lines_by_thread_uuid[uuid] << line
end
lines_by_thread_uuid
end
# @return
def self.non_debugger_thread_list
Thread.list.reject { |thread|
# don't do `is_a? Debugger::DebugThread` because it requires Debugger::DebugThread to be loaded, which it
# won't when not debugging.
thread.class.name == 'Debugger::DebugThread'
}
end
end
end
end
end
end

65
msfcli
View File

@ -16,10 +16,20 @@ require 'rex'
class Msfcli
#
# Attributes
#
# @attribute framework
# @return [Msf::Simple::Framework]
#
# initialize
#
def initialize(args)
@args = {}
@indent = ' '
@framework = nil
@args[:module_name] = args.shift # First argument should be the module name
@args[:mode] = args.pop || 'h' # Last argument should be the mode
@ -31,6 +41,28 @@ class Msfcli
end
end
#
# Instance Methods
#
# The framework to create and list modules.
#
# @return [Msf::Simple::Framework]
def framework
@framework ||= Msf::Simple::Framework.create({'DeferModuleLoads'=>true})
end
# Sets {#framework}.
#
# @raise [ArgumentError] if {#framework} already set
def framework=(framework)
if instance_variable_defined? :@framework
fail ArgumentError.new("framework already set")
end
@framework = framework
end
#
# Returns a usage Rex table
#
@ -73,7 +105,7 @@ class Msfcli
# msfcli will end up loading EVERYTHING to memory to show you a help
# menu plus a list of modules available. Really expensive if you ask me.
$stdout.puts "[*] Please wait while we load the module tree..."
framework = Msf::Simple::Framework.create
self.framework = Msf::Simple::Framework.create
ext = ''
tbl = Rex::Ui::Text::Table.new(
@ -283,7 +315,6 @@ class Msfcli
# Initializes exploit/payload/encoder/nop modules.
#
def init_modules
@framework = Msf::Simple::Framework.create({'DeferModuleLoads'=>true})
$stdout.puts "[*] Initializing modules..."
module_name = @args[:module_name]
@ -297,11 +328,11 @@ class Msfcli
whitelist = generate_whitelist
# Load up all the possible modules, this is where things get slow again
@framework.init_module_paths({:whitelist=>whitelist})
if (@framework.modules.module_load_error_by_path.length > 0)
framework.init_module_paths({:whitelist=>whitelist})
if (framework.modules.module_load_error_by_path.length > 0)
print("Warning: The following modules could not be loaded!\n\n")
@framework.modules.module_load_error_by_path.each do |path, error|
framework.modules.module_load_error_by_path.each do |path, error|
print("\t#{path}: #{error}\n\n")
end
@ -310,16 +341,16 @@ class Msfcli
# Determine what type of module it is
if module_name =~ /exploit\/(.*)/
modules[:module] = @framework.exploits.create($1)
modules[:module] = framework.exploits.create($1)
elsif module_name =~ /auxiliary\/(.*)/
modules[:module] = @framework.auxiliary.create($1)
modules[:module] = framework.auxiliary.create($1)
elsif module_name =~ /post\/(.*)/
modules[:module] = @framework.post.create($1)
modules[:module] = framework.post.create($1)
else
modules[:module] = @framework.exploits.create(module_name)
modules[:module] = framework.exploits.create(module_name)
if modules[:module].nil?
# Try falling back on aux modules
modules[:module] = @framework.auxiliary.create(module_name)
modules[:module] = framework.auxiliary.create(module_name)
end
end
@ -342,7 +373,7 @@ class Msfcli
# Create the payload to use
if (modules[:module].datastore['PAYLOAD'])
modules[:payload] = @framework.payloads.create(modules[:module].datastore['PAYLOAD'])
modules[:payload] = framework.payloads.create(modules[:module].datastore['PAYLOAD'])
if modules[:payload]
modules[:payload].datastore.import_options_from_s(@args[:params].join('_|_'), '_|_')
end
@ -350,7 +381,7 @@ class Msfcli
# Create the encoder to use
if modules[:module].datastore['ENCODER']
modules[:encoder] = @framework.encoders.create(modules[:module].datastore['ENCODER'])
modules[:encoder] = framework.encoders.create(modules[:module].datastore['ENCODER'])
if modules[:encoder]
modules[:encoder].datastore.import_options_from_s(@args[:params].join('_|_'), '_|_')
end
@ -358,7 +389,7 @@ class Msfcli
# Create the NOP to use
if modules[:module].datastore['NOP']
modules[:nop] = @framework.nops.create(modules[:module].datastore['NOP'])
modules[:nop] = framework.nops.create(modules[:module].datastore['NOP'])
if modules[:nop]
modules[:nop].datastore.import_options_from_s(@args[:params].join('_|_'), '_|_')
end
@ -454,7 +485,7 @@ class Msfcli
Msf::Ui::Console::Driver::DefaultPrompt,
Msf::Ui::Console::Driver::DefaultPromptChar,
{
'Framework' => @framework,
'Framework' => framework,
# When I use msfcli, chances are I want speed, so ASCII art fanciness
# probably isn't much of a big deal for me.
'DisableBanner' => true
@ -474,7 +505,7 @@ class Msfcli
con.run_single("exploit")
# If we have sessions or jobs, keep running
if @framework.sessions.length > 0 or @framework.jobs.length > 0
if framework.sessions.length > 0 or framework.jobs.length > 0
con.run
else
con.run_single("quit")
@ -558,7 +589,7 @@ class Msfcli
end
# Process special var/val pairs...
Msf::Ui::Common.process_cli_arguments(@framework, @args[:params])
Msf::Ui::Common.process_cli_arguments(framework, @args[:params])
engage_mode(modules)
$stdout.puts

11
spec/lib/msf/core/framework_spec.rb
View File

@ -4,24 +4,25 @@ require 'spec_helper'
require 'msf/core/framework'
describe Msf::Framework do
include_context 'Msf::Framework#threads cleaner'
describe "#version" do
CURRENT_VERSION = "4.10.1-dev"
subject do
subject(:framework) do
described_class.new
end
it "should return the current version" do
subject.version.should == CURRENT_VERSION
framework.version.should == CURRENT_VERSION
end
it "should return the Version constant" do
described_class.const_get(:Version).should == subject.version
described_class.const_get(:Version).should == framework.version
end
it "should return the concatenation of Major.Minor.Point-Release" do
major,minor,point,release = subject.version.split(/[.-]/)
major,minor,point,release = framework.version.split(/[.-]/)
major.to_i.should == described_class::Major
minor.to_i.should == described_class::Minor
point.to_i.should == described_class::Point
@ -30,7 +31,7 @@ describe Msf::Framework do
skip "conform to SemVer 2.0 syntax: http://semver.org/" do
it "should have constants that correspond to SemVer standards" do
major,minor,patch,label = subject.version.split(/[.-]/)
major,minor,patch,label = framework.version.split(/[.-]/)
major.to_i.should == described_class::VERSION::MAJOR
minor.to_i.should == described_class::VERSION::MINOR
point.to_i.should == described_class::VERSION::POINT

5
spec/lib/msf/core/task_manager_spec.rb
View File

@ -4,10 +4,7 @@ require 'msf/core'
require 'msf/core/task_manager'
describe Msf::TaskManager do
let(:framework) do
Msf::Framework.new
end
include_context 'Msf::Simple::Framework'
let(:tm) do
Msf::TaskManager.new(framework)

1099
spec/msfcli_spec.rb
View File

File diff suppressed because it is too large Load Diff

3
spec/spec_helper.rb
View File

@ -20,6 +20,8 @@ require 'rspec/rails/fixture_support'
require 'rspec/rails/matchers'
require 'rspec/rails/mocks'
require 'metasploit/framework/spec'
FILE_FIXTURES_PATH = File.expand_path(File.dirname(__FILE__)) + '/file_fixtures/'
# Load the shared examples from the following engines
@ -57,3 +59,4 @@ RSpec.configure do |config|
end
Metasploit::Framework::Spec::Constants::Suite.configure!
Metasploit::Framework::Spec::Threads::Suite.configure!

16
spec/support/shared/contexts/msf/framework/threads/cleaner.rb Normal file
View File

@ -0,0 +1,16 @@
shared_context 'Msf::Framework#threads cleaner' do
after(:each) do
# explicitly kill threads so that they don't exhaust connection pool
thread_manager = framework.threads
thread_manager.each do |thread|
thread.kill
# ensure killed thread is cleaned up by VM
thread.join
end
thread_manager.monitor.kill
# ensure killed thread is cleaned up by VM
thread_manager.monitor.join
end
end

17
spec/support/shared/contexts/msf/simple/framework.rb
View File

@ -3,6 +3,8 @@ require 'msf/base/simple/framework'
require 'metasploit/framework'
shared_context 'Msf::Simple::Framework' do
include_context 'Msf::Framework#threads cleaner'
let(:dummy_pathname) do
Rails.root.join('spec', 'dummy')
end
@ -26,19 +28,4 @@ shared_context 'Msf::Simple::Framework' do
after(:each) do
dummy_pathname.rmtree
end
after(:each) do
# explicitly kill threads so that they don't exhaust connection pool
thread_manager = framework.threads
thread_manager.each do |thread|
thread.kill
# ensure killed thread is cleaned up by VM
thread.join
end
thread_manager.monitor.kill
# ensure killed thread is cleaned up by VM
thread_manager.monitor.join
end
end