Land #4332, test optimization for Cucumber

* Make Cuke run faster on TravisCI

.travis.yml

@@ -1,14 +1,14 @@
+bundler_args: --without development coverage
 env:
-  - RAKE_TASK=cucumber
-# Commenting out the boot tests due to chronic timeouts.
-#  - RAKE_TASK=cucumber:boot
-  - RAKE_TASK=spec SPEC_OPTS="--tag content"
-  - RAKE_TASK=spec SPEC_OPTS="--tag ~content"
+  - RAKE_TASKS="cucumber cucumber:boot"
+  - RAKE_TASKS=spec SPEC_OPTS="--tag content"
+  - RAKE_TASKS=spec SPEC_OPTS="--tag ~content"
 
 language: ruby
 matrix:
   fast_finish: true
 before_install:
+  - "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
   - rake --version
   - sudo apt-get update -qq
   - sudo apt-get install -qq libpcap-dev
@@ -25,7 +25,7 @@ before_script:
   - bundle exec rake db:migrate
 script:
   # fail build if db/schema.rb update is not committed
-  - git diff --exit-code && bundle exec rake $RAKE_TASK
+  - git diff --exit-code && bundle exec rake $RAKE_TASKS
 
 rvm:
   - '1.9.3'

Gemfile

@@ -3,6 +3,14 @@ source 'https://rubygems.org'
 #   spec.add_runtime_dependency '<name>', [<version requirements>]
 gemspec name: 'metasploit-framework'
 
+# separate from test as simplecov is not run on travis-ci
+group :coverage do
+  # code coverage for tests
+  # any version newer than 0.5.4 gives an Encoding error when trying to read the source files.
+  # see: https://github.com/colszowka/simplecov/issues/127 (hopefully fixed in 0.8.0)
+  gem 'simplecov', '0.5.4', :require => false
+end
+
 group :db do
   gemspec name: 'metasploit-framework-db'
 end
@@ -44,10 +52,6 @@ group :test do
   # cucumber + automatic database cleaning with database_cleaner
   gem 'cucumber-rails', :require => false
   gem 'shoulda-matchers'
-  # code coverage for tests
-  # any version newer than 0.5.4 gives an Encoding error when trying to read the source files.
-  # see: https://github.com/colszowka/simplecov/issues/127 (hopefully fixed in 0.8.0)
-  gem 'simplecov', '0.5.4', :require => false
   # Manipulate Time.now in specs
   gem 'timecop'
 end

config/application.rb

@@ -9,6 +9,7 @@ all_environments = [
 
 Bundler.require(
     *Rails.groups(
+        coverage: [:test],
         db: all_environments,
         pcap: all_environments
     )

features/commands/help.feature

@@ -1,12 +1,9 @@
 Feature: Help command
 
   Background:
-    Given I run `msfconsole` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
+    Given I run `msfconsole --defer-module-loads -x help -x exit`
 
   Scenario: The 'help' command's output
-    When I type "help"
-    And I type "exit"
     Then the output should contain:
       """
       Core Commands

features/msfconsole/database_yml.feature

@@ -48,9 +48,7 @@ Feature: `msfconsole` `database.yml`
         database: project_metasploit_framework_test
         username: project_metasploit_framework_test
       """
-    When I run `msfconsole --environment test --yaml command_line.yml` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --environment test --execute-command exit --yaml command_line.yml`
     Then the output should contain "command_line_metasploit_framework_test"
 
   Scenario: Without --yaml, MSF_DATABASE_CONFIG wins
@@ -84,9 +82,7 @@ Feature: `msfconsole` `database.yml`
         database: project_metasploit_framework_test
         username: project_metasploit_framework_test
       """
-    When I run `msfconsole --environment test` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --environment test --execute-command exit`
     Then the output should contain "environment_metasploit_framework_test"
 
   Scenario: Without --yaml or MSF_DATABASE_CONFIG, ~/.msf4/database.yml wins
@@ -113,9 +109,7 @@ Feature: `msfconsole` `database.yml`
         database: project_metasploit_framework_test
         username: project_metasploit_framework_test
       """
-    When I run `msfconsole --environment test` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --environment test --execute-command exit`
     Then the output should contain "user_metasploit_framework_test"
 
   Scenario: Without --yaml, MSF_DATABASE_CONFIG or ~/.msf4/database.yml, project "database.yml" wins
@@ -133,9 +127,7 @@ Feature: `msfconsole` `database.yml`
         database: project_metasploit_framework_test
         username: project_metasploit_framework_test
       """
-    When I run `msfconsole --environment test` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --environment test --execute-command exit`
     Then the output should contain "project_metasploit_framework_test"
 
 
@@ -148,10 +140,7 @@ Feature: `msfconsole` `database.yml`
     And a mocked home directory
     And I cd to "../.."
     And the project "database.yml" does not exist
-    When I run `msfconsole --environment test` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    And I type "db_status"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --environment test --execute-command db_status --execute-command exit`
     Then the output should not contain "command_line_metasploit_framework_test"
     And the output should not contain "environment_metasploit_framework_test"
     And the output should not contain "user_metasploit_framework_test"
@@ -159,9 +148,6 @@ Feature: `msfconsole` `database.yml`
     And the output should contain "[*] postgresql selected, no connection"
 
   Scenario: Starting `msfconsole` with a valid database.yml
-    Given I run `msfconsole` interactively
-    And I wait for stdout to contain "Free Metasploit Pro trial: http://r-7.co/trymsp"
-    When I type "db_status"
-    And I type "exit"
+    When I run `msfconsole --defer-module-loads --execute-command db_status --execute-command exit`
     Then the output should contain "[*] postgresql connected to metasploit_framework_test"
 

features/support/env.rb

@@ -1,5 +1,5 @@
-# Has to be the first file required so that all other files show coverage information
-require 'simplecov'
+# @note `require 'simplecov'` is not used here because all features currently use external `msfconsole` process, so only
+#   that child process needs to load 'simplecov'.
 
 # IMPORTANT: This file is generated by cucumber-rails - edit at your own peril.
 # It is recommended to regenerate this file in the future when you upgrade to a 

features/support/hooks.rb

@@ -4,23 +4,26 @@ Before do
   @aruba_timeout_seconds = 8.minutes
 end
 
-Before do |scenario|
-  command_name = case scenario
-                 when Cucumber::Ast::Scenario, Cucumber::Ast::ScenarioOutline
-                   "#{scenario.feature.title} #{scenario.name}"
-                 when Cucumber::Ast::OutlineTable::ExampleRow
-                   scenario_outline = scenario.scenario_outline
+# don't setup child processes to load simplecov_setup.rb if simplecov isn't installed
+unless Bundler.settings.without.include?(:coverage)
+  Before do |scenario|
+    command_name = case scenario
+                   when Cucumber::Ast::Scenario, Cucumber::Ast::ScenarioOutline
+                     "#{scenario.feature.title} #{scenario.name}"
+                   when Cucumber::Ast::OutlineTable::ExampleRow
+                     scenario_outline = scenario.scenario_outline
 
-                   "#{scenario_outline.feature.title} #{scenario_outline.name} #{scenario.name}"
-                 else
-                   raise TypeError, "Don't know how to extract command name from #{scenario.class}"
-                 end
+                     "#{scenario_outline.feature.title} #{scenario_outline.name} #{scenario.name}"
+                   else
+                     raise TypeError, "Don't know how to extract command name from #{scenario.class}"
+                   end
 
-  # Used in simplecov_setup so that each scenario has a different name and their coverage results are merged instead
-  # of overwriting each other as 'Cucumber Features'
-  set_env('SIMPLECOV_COMMAND_NAME', command_name)
+    # Used in simplecov_setup so that each scenario has a different name and their coverage results are merged instead
+    # of overwriting each other as 'Cucumber Features'
+    set_env('SIMPLECOV_COMMAND_NAME', command_name)
 
-  simplecov_setup_pathname = Pathname.new(__FILE__).expand_path.parent.join('simplecov_setup')
-  # set environment variable so child processes will merge their coverage data with parent process's coverage data.
-  set_env('RUBYOPT', "-r#{simplecov_setup_pathname} #{ENV['RUBYOPT']}")
+    simplecov_setup_pathname = Pathname.new(__FILE__).expand_path.parent.join('simplecov_setup')
+    # set environment variable so child processes will merge their coverage data with parent process's coverage data.
+    set_env('RUBYOPT', "#{ENV['RUBYOPT']} -r#{simplecov_setup_pathname}")
+  end
 end

features/support/simplecov_setup.rb

@@ -1,11 +1,16 @@
 # @note this file is loaded in env.rb to setup simplecov using RUBYOPTs for child processes
 
-require 'simplecov'
+simplecov_command_name = ENV['SIMPLECOV_COMMAND_NAME']
 
-require 'pathname'
+# will not be set if hook does not run because `bundle install --without coverage`
+if simplecov_command_name
+  require 'simplecov'
 
-root = Pathname(__FILE__).expand_path.parent.parent.parent
+  require 'pathname'
 
-SimpleCov.command_name(ENV['SIMPLECOV_COMMAND_NAME'])
-SimpleCov.root(root)
-load root.join('.simplecov')
+  root = Pathname(__FILE__).expand_path.parent.parent.parent
+
+  SimpleCov.command_name(simplecov_command_name)
+  SimpleCov.root(root)
+  load root.join('.simplecov')
+end

lib/metasploit/framework/command/console.rb

@@ -61,6 +61,7 @@ class Metasploit::Framework::Command::Console < Metasploit::Framework::Command::
       driver_options['DatabaseEnv'] = options.environment
       driver_options['DatabaseMigrationPaths'] = options.database.migrations_paths
       driver_options['DatabaseYAML'] = options.database.config
+      driver_options['DeferModuleLoads'] = options.modules.defer_loads
       driver_options['Defanged'] = options.console.defanged
       driver_options['DisableBanner'] = options.console.quiet
       driver_options['DisableDatabase'] = options.database.disable

lib/metasploit/framework/parsed_options/base.rb

@@ -85,6 +85,7 @@ class Metasploit::Framework::ParsedOptions::Base
       options.framework.config = nil
 
       options.modules = ActiveSupport::OrderedOptions.new
+      options.modules.defer_loads = false
       options.modules.path = nil
 
       @options = options
@@ -155,6 +156,13 @@ class Metasploit::Framework::ParsedOptions::Base
       option_parser.separator ''
       option_parser.separator 'Module options'
 
+      option_parser.on(
+          '--defer-module-loads',
+          'Defer module loading unless explicitly asked.'
+      ) do
+        options.modules.defer_loads = true
+      end
+
       option_parser.on(
           '-m',
           '--module-path DIRECTORY',

lib/msf/base/simple/framework.rb

@@ -62,21 +62,29 @@ module Framework
       Msf::MODULE_POST    => Msf::Simple::Post,
     }
 
-  #
   # Create a simplified instance of the framework.  This routine takes a hash
   # of parameters as an argument.  This hash can contain:
   #
-  #   OnCreateProc => A callback procedure that is called once the framework
-  #   instance is created.
-  #
+  # @param opts [Hash{String => Object}]
+  # @option opts (see simplify)
+  # @return [Msf::Simple::Frameworkt s]
   def self.create(opts = {})
     framework = Msf::Framework.new(opts)
     return simplify(framework, opts)
   end
 
+  # @note If `opts['ConfigDirectory']` is set, then `Msf::Config::Defaults['ConfigDirectory']` will be updated to
+  #   `opts['ConfigDirectory']`.
   #
   # Extends a framework object that may already exist.
   #
+  # @param framework [Msf::Framework, Msf::Simple::Framework] framework to simplify
+  # @param opts [Hash{String => Object}]
+  # @option opts [#call] 'OnCreateProc' Proc to call after {#init_simplified}.  Will be passed `framework`.
+  # @option opts [String] 'ConfigDirectory'  Directory where configuration is saved.  The `~/.msf4` directory.
+  # @option opts [Boolean] 'DisableLogging' (false) `true` to disable `Msf::Logging.init`
+  # @option opts [Boolean] 'DeferModuleLoads' (false) `true` to disable `framework.init_module_paths`.
+  # @return [Msf::Simple::Framework] `framework`
   def self.simplify(framework, opts)
 
     # If the framework instance has not already been extended, do it now.

lib/msf/ui/console/driver.rb

@@ -61,7 +61,7 @@ class Driver < Msf::Ui::Driver
     # Initialize attributes
 
     # Defer loading of modules until paths from opts can be added below
-    framework_create_options = {'DeferModuleLoads' => true}.merge(opts)
+    framework_create_options = opts.merge('DeferModuleLoads' => true)
     self.framework = opts['Framework'] || Msf::Simple::Framework.create(framework_create_options)
 
     if self.framework.datastore['Prompt']
@@ -187,10 +187,12 @@ class Driver < Msf::Ui::Driver
       # framework.db.active will be true if after_establish_connection ran directly when connection_established? was
       # already true or if framework.db.connect called after_establish_connection.
       if framework.db.active
-        self.framework.modules.refresh_cache_from_database
+        unless opts['DeferModuleLoads']
+          self.framework.modules.refresh_cache_from_database
 
-        if self.framework.modules.cache_empty?
-          print_status("The initial module cache will be built in the background, this can take 2-5 minutes...")
+          if self.framework.modules.cache_empty?
+            print_status("The initial module cache will be built in the background, this can take 2-5 minutes...")
+          end
         end
       elsif !framework.db.error.nil?
         if framework.db.error.to_s =~ /RubyGem version.*pg.*0\.11/i
@@ -212,8 +214,8 @@ class Driver < Msf::Ui::Driver
       end
     end
 
-    # Initialize the module paths only if we didn't get passed a Framework instance
-    unless opts['Framework']
+    # Initialize the module paths only if we didn't get passed a Framework instance and 'DeferModuleLoads' is false
+    unless opts['Framework'] || opts['DeferModuleLoads']
       # Configure the framework module paths
       self.framework.init_module_paths
       self.framework.modules.add_module_path(opts['ModulePath']) if opts['ModulePath']

spec/spec_helper.rb

@@ -1,7 +1,9 @@
 # -*- coding: binary -*-
 ENV['RAILS_ENV'] = 'test'
 
-require 'simplecov'
+unless Bundler.settings.without.include?(:coverage)
+  require 'simplecov'
+end
 
 # @note must be before loading config/environment because railtie needs to be loaded before
 #   `Metasploit::Framework::Application.initialize!` is called.