From f4580317982f91bbce6dd0cf47f0ea0ae123d43b Mon Sep 17 00:00:00 2001 From: Matthew Kienow Date: Wed, 1 Aug 2018 02:29:39 -0400 Subject: [PATCH] Add enhanced msfdb with web service support Derived from the msfdb script in the metasploit-omnibus repo. --- msfdb | 767 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 767 insertions(+) create mode 100755 msfdb diff --git a/msfdb b/msfdb new file mode 100755 index 0000000000..df0258f09b --- /dev/null +++ b/msfdb @@ -0,0 +1,767 @@ +#!/usr/bin/env ruby + +require 'fileutils' +require 'json' +require 'net/http' +require 'net/https' +require 'open3' +require 'optparse' +require 'rex/socket' +require 'rex/text' +require 'sysrandom/securerandom' +require 'uri' +require 'yaml' + +@script_name = File.basename(__FILE__) +@framework = File.expand_path(File.dirname(__FILE__)) + +@localconf = "#{ENV['HOME']}/.msf4" +@db = "#{@localconf}/db" +@db_conf = "#{@localconf}/database.yml" + +@ws_tag = 'msf-ws' +@ws_conf = "#{@localconf}/#{@ws_tag}-config.ru" +@ws_ssl_key_default = "#{@localconf}/#{@ws_tag}-key.pem" +@ws_ssl_cert_default = "#{@localconf}/#{@ws_tag}-cert.pem" +@ws_log = "#{@localconf}/logs/#{@ws_tag}.log" +@ws_pid = "#{@localconf}/#{@ws_tag}.pid" + +@current_user = ENV['LOGNAME'] || ENV['USERNAME'] || ENV['USER'] +@msf_ws_user = (@current_user || "msfadmin").to_s.strip +@ws_generated_ssl = false +@ws_api_token = nil + +@components = %w[database webservice] + +@options = { + component: :all, + debug: false, + msf_db_name: 'msf', + msf_db_user: 'msf', + msftest_db_name: 'msftest', + msftest_db_user: 'msftest', + db_port: 5433, + db_pool: 200, + address: 'localhost', + port: 8080, + ssl: true, + ssl_cert: @ws_ssl_cert_default, + ssl_key: @ws_ssl_key_default, + ssl_disable_verify: false, + ws_env: ENV['RACK_ENV'] || 'production', + retry_max: 10, + retry_delay: 5.0, + ws_user: nil +} + + +def run_cmd(cmd, input = nil) + exitstatus = 0 + err = out = "" + + puts "run_cmd: cmd=#{cmd}, input=#{input}" if @options[:debug] + + Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr| + stdin.puts(input) if input + if @options[:debug] + err = stderr.read + out = stdout.read + end + exitstatus = wait_thr.value.exitstatus + end + + if exitstatus != 0 + if @options[:debug] + puts "'#{cmd}' returned #{exitstatus}" + puts out + puts err + end + end + + exitstatus +end + +def run_psql(cmd, db_name: 'postgres') + if @options[:debug] + puts "psql -p #{@options[:db_port]} -c \"#{cmd};\" #{db_name}" + end + + run_cmd("psql -p #{@options[:db_port]} -c \"#{cmd};\" #{db_name}") +end + +def pw_gen + SecureRandom.base64(32) +end + +def tail(file) + begin + File.readlines(file).last.to_s.strip + rescue + nil + end +end + +def status_db + update_db_port + + if Dir.exist?(@db) + if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0 + puts "Database started at #{@db}" + else + puts "Database is not running at #{@db}" + end + else + puts "No database found at #{@db}" + end +end + +def started_db + if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0 + puts "Database already started at #{@db}" + return true + end + + print "Starting database at #{@db}..." + run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} -l #{@db}/log start") + sleep(2) + if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") != 0 + puts 'failed' + false + else + puts 'success' + true + end +end + +def start_db + if !Dir.exist?(@db) + puts "No database found at #{@db}, not starting" + return + end + + update_db_port + + while !started_db + last_log = tail("#{@db}/log") + puts last_log + fixed = false + if last_log =~ /not compatible/ + puts "Please attempt to upgrade the database manually using pg_upgrade." + end + if !fixed + if ask_yn('If your database is corrupt, would you to reinitialize it?') + fixed = reinit_db + end + end + if !fixed + if !ask_yn('Database not started, try again?') + return + end + end + end +end + +def stop_db + update_db_port + + if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0 + puts "Stopping database at #{@db}" + run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} stop") + else + puts "Database is no longer running at #{@db}" + end +end + +def restart_db + stop_db + start_db +end + +def create_db + puts "Creating database at #{@db}" + Dir.mkdir(@db) + run_cmd("initdb --auth-host=trust --auth-local=trust -E UTF8 #{@db}") + + File.open("#{@db}/pg_hba.conf", 'w') do |f| + f.puts "host \"msf\" \"#{@options[:msf_db_user]}\" 127.0.0.1/32 md5" + f.puts "host \"msftest\" \"#{@options[:msftest_db_user]}\" 127.0.0.1/32 md5" + f.puts "host \"postgresql\" \"#{@options[:msftest_db_user]}\" 127.0.0.1/32 md5" + f.puts "host \"template1\" all 127.0.0.1/32 trust" + if Gem.win_platform? + f.puts "host all all 127.0.0.1/32 trust" + f.puts "host all all ::1/128 trust" + else + f.puts "local all all trust" + end + end + + File.open("#{@db}/postgresql.conf", 'a') do |f| + f.puts "port = #{@options[:db_port]}" + end +end + +def init_db + if Dir.exist?(@db) + puts "Found a database at #{@db}, checking to see if it is started" + start_db + return + end + + if File.exist?(@db_conf) + if !ask_yn("Found database config at #{@db_conf}, do you want to overwrite it?") + return + end + end + + # Generate new database passwords + msf_pass = pw_gen + msftest_pass = pw_gen + + # Write a default database config file + Dir.mkdir(@localconf) unless File.directory?(@localconf) + File.open(@db_conf, 'w') do |f| + f.puts <<~EOF + development: &pgsql + adapter: postgresql + database: #{@options[:msf_db_name]} + username: #{@options[:msf_db_user]} + password: #{msf_pass} + host: 127.0.0.1 + port: #{@options[:db_port]} + pool: #{@options[:db_pool]} + + production: &production + <<: *pgsql + + test: + <<: *pgsql + database: #{@options[:msftest_db_name]} + username: #{@options[:msftest_db_user]} + password: #{msftest_pass} + EOF + end + + File.chmod(0640, @db_conf) + + create_db + + start_db + + puts 'Creating database users' + run_psql("create user #{@options[:msf_db_user]} with password '#{msf_pass}'") + run_psql("create user #{@options[:msftest_db_user]} with password '#{msftest_pass}'") + run_psql("alter role #{@options[:msf_db_user]} createdb") + run_psql("alter role #{@options[:msftest_db_user]} createdb") + run_psql("alter role #{@options[:msf_db_user]} with password '#{msf_pass}'") + run_psql("alter role #{@options[:msftest_db_user]} with password '#{msftest_pass}'") + run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msf_db_user]} -h 127.0.0.1 -U #{@options[:msf_db_user]} -E UTF-8 -T template0 #{@options[:msf_db_name]}", + "#{msf_pass}\n#{msf_pass}\n") + run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msftest_db_user]} -h 127.0.0.1 -U #{@options[:msftest_db_user]} -E UTF-8 -T template0 #{@options[:msftest_db_name]}", + "#{msftest_pass}\n#{msftest_pass}\n") + + puts 'Creating initial database schema' + Dir.chdir(@framework) do + run_cmd("bundle exec rake db:migrate") + end +end + +def update_db_port + if File.file?(@db_conf) + config = YAML.load(File.read(@db_conf)) + if config["production"] && config["production"]["port"] + port = config["production"]["port"] + if port != @options[:db_port] + puts "Using database port #{port} found in #{@db_conf}" + @options[:db_port] = port + end + end + end +end + +def ask_yn(question) + loop do + print "#{question}: " + yn = STDIN.gets + case yn + when /^[Yy]/ + return true + when /^[Nn]/ + return false + else + puts 'Please answer yes or no.' + end + end +end + +def ask_value(question, default_value) + print "#{question}[#{default_value}]: " + input = STDIN.gets.strip + if input.nil? || input.empty? + return default_value + else + return input + end +end + + +def delete_db + if Dir.exist?(@db) + puts "Deleting all data at #{@db}" + stop_db + FileUtils.rm_rf(@db) + if File.exist?(@db_conf) && ask_yn("Delete database configuration at #{@db_conf}?") + File.delete(@db_conf) + end + else + puts "No data at #{@db}, doing nothing" + end +end + +def reinit_db + delete_db + init_db +end + +def status_web_service + if File.file?(@ws_pid) + ws_pid = tail(@ws_pid) + if ws_pid.nil? || !process_active?(ws_pid.to_i) + puts "MSF web service PID file found, but no active process running as PID #{ws_pid}" + else + puts "MSF web service is running as PID #{ws_pid}" + end + else + puts "No MSF web service PID file found at #{@ws_pid}" + end +end + +def init_web_service + if File.file?(@ws_conf) + if !ask_yn("Found web service config at #{@ws_conf}, do you want to overwrite it?") + return + end + end + + if @options[:ws_user].nil? + @msf_ws_user = ask_value("Initial MSF web service account username?", @msf_ws_user) + else + @msf_ws_user = @options[:ws_user] + end + + # Write a default Rack config file for the web service + Dir.mkdir(@localconf) unless File.directory?(@localconf) + File.open(@ws_conf, 'w') do |f| + f.puts <<~EOF + # #{File.basename(@ws_conf)} + # created on: #{Time.now.utc} + lib_path = File.expand_path('./lib/', '#{@framework}') + $LOAD_PATH << lib_path unless $LOAD_PATH.include?(lib_path) + require 'msf/core/db_manager/http/metasploit_api_app' + run MetasploitApiApp + EOF + end + File.chmod(0640, @ws_conf) + + if @options[:ssl] && ((!File.file?(@options[:ssl_key]) || !File.file?(@options[:ssl_cert])) || + (@options[:ssl_key] == @ws_ssl_key_default && @options[:ssl_cert] == @ws_ssl_cert_default)) + generate_web_service_ssl(key: @options[:ssl_key], cert: @options[:ssl_cert]) + end + + if !start_web_service + return + end + + # wait until web service is online + retry_count = 0 + is_online = web_service_online + while !is_online && retry_count < @options[:retry_max] + retry_count += 1 + if @options[:debug] + puts "MSF web service doesn't appear to be online. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}" + end + sleep(@options[:retry_delay]) + is_online = web_service_online + end + + if is_online + add_web_service_user + else + puts "MSF web service does not appear to be online; aborting initialize." + end +end + +def start_web_service + unless File.file?(@ws_conf) + puts "No MSF web service configuration found at #{@ws_conf}, not starting" + return false + end + + # daemonize MSF web service + puts "Starting MSF web service" + if run_cmd("#{thin_cmd} start") == 0 + puts "MSF web service started" + return true + else + puts "MSF web service not started" + return false + end +end + +def stop_web_service + ws_pid = tail(@ws_pid) + if ws_pid.nil? || !process_active?(ws_pid.to_i) + puts "MSF web service is no longer running" + if File.file?(@ws_pid) + puts "Deleting MSF web service PID file #{@ws_pid}" + File.delete(@ws_pid) + end + else + puts "Stopping MSF web service PID #{ws_pid}" + run_cmd("#{thin_cmd} stop") + end +end + +def restart_web_service + stop_web_service + start_web_service +end + +def delete_web_service + stop_web_service + if File.file?(@ws_conf) && ask_yn("Delete MSF web service configuration at #{@ws_conf}?") + File.delete(@ws_conf) + end +end + +def reinit_web_service + delete_web_service + init_web_service +end + +def generate_web_service_ssl(key:, cert:) + @ws_generated_ssl = true + if (File.file?(key) || File.file?(cert)) && + !ask_yn("Either MSF web service SSL key #{key} or certificate #{cert} already exist, overwrite both?") + return + end + + puts "Generating SSL key and certificate for MSF web service" + # @ssl_cert = Rex::Socket::SslTcpServer.ssl_generate_certificate + @ssl_key, @ssl_cert, @ssl_extra_chain_cert = Rex::Socket::Ssl.ssl_generate_certificate + + # write PEM format key and certificate + mode = 'wb' + mode_int = 0600 + File.open(key, mode) { |f| f.write(@ssl_key.to_pem) } + File.chmod(mode_int, key) + + File.open(cert, mode) { |f| f.write(@ssl_cert.to_pem) } + File.chmod(mode_int, cert) +end + +def web_service_online + msf_version_uri = get_web_service_uri(path: '/api/v1/msf/version') + response = http_request(uri: msf_version_uri, method: :get, + skip_verify: skip_ssl_verify, cert: get_ssl_cert) + puts "web_service_online: response=#{response}" if @options[:debug] + !response.nil? && !response.dig(:data, :metasploit_version).nil? +end + +def add_web_service_user + puts "Creating MSF web service user #{@msf_ws_user}" + + # Generate new web service user password + msf_ws_pass = pw_gen + cred_data = { username: @msf_ws_user, password: msf_ws_pass } + + # Send request to create new admin user + user_data = cred_data.merge({ admin: true }) + user_uri = get_web_service_uri(path: '/api/v1/users') + response = http_request(uri: user_uri, data: user_data, method: :post, + skip_verify: skip_ssl_verify, cert: get_ssl_cert) + puts "add_web_service_user: create user response=#{response}" if @options[:debug] + if response.nil? || response.dig(:data, :username) != @msf_ws_user + puts "Error creating MSF web service user #{@msf_ws_user}" + return false + end + puts "\nMSF web service user: #{@msf_ws_user}" + puts "MSF web service password: #{msf_ws_pass}" + + # Send request to create new API token for the user + generate_token_uri = get_web_service_uri(path: '/api/v1/auth/generate-token') + response = http_request(uri: generate_token_uri, query: cred_data, method: :get, + skip_verify: skip_ssl_verify, cert: get_ssl_cert) + puts "add_web_service_user: generate token response=#{response}" if @options[:debug] + if response.nil? || (@ws_api_token = response.dig(:data, :token)).nil? + puts "Error creating MSF web service user API token" + return false + end + puts "MSF web service user API token: #{@ws_api_token}" + puts "Please store these credentials securely." + return true +end + +def get_web_service_uri(path: nil) + uri_class = @options[:ssl] ? URI::HTTPS : URI::HTTP + uri_class.build({host: get_web_service_host, port: @options[:port], path: path}) +end + +def get_web_service_host + # user specified any address INADDR_ANY (0.0.0.0), return a routable address + @options[:address] == '0.0.0.0' ? 'localhost' : @options[:address] +end + +def skip_ssl_verify + @ws_generated_ssl || @options[:ssl_disable_verify] +end + +def get_ssl_cert + @options[:ssl] ? @options[:ssl_cert] : nil +end + +def thin_cmd + server_opts = "--rackup #{@ws_conf} --address #{@options[:address]} --port #{@options[:port]}" + ssl_opts = @options[:ssl] ? "--ssl --ssl-key-file #{@options[:ssl_key]} --ssl-cert-file #{@options[:ssl_cert]}" : '' + ssl_opts << ' --ssl-disable-verify' if skip_ssl_verify + adapter_opts = "--environment #{@options[:ws_env]}" + daemon_opts = "--daemonize --log #{@ws_log} --pid #{@ws_pid} --tag #{@ws_tag}" + all_opts = [server_opts, ssl_opts, adapter_opts, daemon_opts].reject(&:empty?).join(' ') + + "thin #{all_opts}" +end + +def process_active?(pid) + begin + # group_id = Process.getpgid(pid) + Process.kill(0, pid) + true + rescue Errno::ESRCH + false + end +end + +def http_request(uri:, query: nil, data: nil, method: :get, skip_verify: false, cert: nil) + headers = { 'User-Agent': @script_name } + query_str = (!query.nil? && !query.empty?) ? URI.encode_www_form(query.compact) : nil + uri.query = query_str + + http = Net::HTTP.new(uri.host, uri.port) + if uri.is_a?(URI::HTTPS) + http.use_ssl = true + if skip_verify + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + else + # https://stackoverflow.com/questions/22093042/implementing-https-certificate-pubkey-pinning-with-ruby + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + user_passed_cert = OpenSSL::X509::Certificate.new(File.read(cert)) + + http.verify_callback = lambda do |preverify_ok, cert_store| + server_cert = cert_store.chain[0] + return true unless server_cert.to_der == cert_store.current_cert.to_der + same_public_key?(server_cert, user_passed_cert) + end + end + end + + begin + case method + when :get + request = Net::HTTP::Get.new(uri.request_uri, initheader=headers) + when :post + request = Net::HTTP::Post.new(uri.request_uri, initheader=headers) + else + raise Exception, "Request method #{method} is not handled" + end + + request.content_type = 'application/json' + unless data.nil? + json_body = data.to_json + request.body = json_body + end + + response = http.request(request) + unless response.body.nil? || response.body.empty? + return JSON.parse(response.body, symbolize_names: true) + end + rescue EOFError => e + puts "No data was returned for HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug] + rescue => e + puts "Problem with HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug] + end +end + +# Tells us whether the private keys on the passed certificates match +# and use the same algo +def same_public_key?(ref_cert, actual_cert) + pkr, pka = ref_cert.public_key, actual_cert.public_key + + # First check if the public keys use the same crypto... + return false unless pkr.class == pka.class + # ...and then - that they have the same contents + return false unless pkr.to_pem == pka.to_pem + + true +end + +def parse_args(args) + subtext = <<~USAGE + Commands: + init initialize the component + reinit delete and reinitialize the component + delete delete and stop the component + status check component status + start start the component + stop stop the component + restart restart the component + USAGE + + parser = OptionParser.new do |opts| + opts.banner = "Usage: #{@script_name} [options] " + opts.separator('Manage a Metasploit Framework database and web service') + opts.separator('') + opts.separator('General Options:') + opts.on("--component COMPONENT", @components + ['all'], "Component used with provided command (default: all)", + " (#{@components.join(', ')})") { |component| + @options[:component] = component.to_sym + } + + opts.on("-d", "--debug", "Enable debug output") { |d| @options[:debug] = d } + + opts.separator('') + opts.separator('Database Options:') + opts.on("--msf-db-name NAME", "Database name (default: #{@options[:msf_db_name]})") { |n| + @options[:msf_db_name] = n + } + + opts.on("--msf-db-user-name USER", "Database username (default: #{@options[:msf_db_user]})") { |u| + @options[:msf_db_user] = u + } + + opts.on("--msf-test-db-name NAME", "Test database name (default: #{@options[:msftest_db_name]})") { |n| + @options[:msftest_db_name] = n + } + + opts.on("--msf-test-db-user-name USER", "Test database username (default: #{@options[:msftest_db_user]})") { |u| + @options[:msftest_db_user] = u + } + + opts.on("--db-port PORT", Integer, "Database port (default: #{@options[:db_port]})") { |p| + @options[:db_port] = p + } + + opts.on("--db-pool MAX", Integer, "Database connection pool size (default: #{@options[:db_pool]})") { |m| + @options[:db_pool] = m + } + + opts.separator('') + opts.separator('Web Service Options:') + opts.on("-a", "--address ADDRESS", + "Bind to host address (default: #{@options[:address]})") { |a| + @options[:address] = a + } + + opts.on("-p", "--port PORT", Integer, + "Web service port (default: #{@options[:port]})") { |p| + @options[:port] = p + } + + opts.on("--[no-]ssl", "Enable SSL (default: #{@options[:ssl]})") { |s| @options[:ssl] = s } + + opts.on("--ssl-key-file PATH", "Path to private key (default: #{@options[:ssl_key]})") { |p| + @options[:ssl_key] = p + } + + opts.on("--ssl-cert-file PATH", "Path to certificate (default: #{@options[:ssl_cert]})") { |p| + @options[:ssl_cert] = p + } + + opts.on("--[no-]ssl-disable-verify", + "Disables (optional) client cert requests (default: #{@options[:ssl_disable_verify]})") { |v| + @options[:ssl_disable_verify] = v + } + + opts.on("--environment ENV", ['production', 'development'], + "Web service framework environment (default: #{@options[:ws_env]})", + " (production, development)") { |e| + @options[:ws_env] = e + } + + opts.on("--retry-max MAX", Integer, + "Maximum number of web service connect attempts (default: #{@options[:retry_max]})") { |m| + @options[:retry_max] = m + } + + opts.on("--retry-delay DELAY", Float, + "Delay in seconds between web service connect attempts (default: #{@options[:retry_delay]})") { |d| + @options[:retry_delay] = d + } + + opts.on("--user USER", "Initial web service admin username (default: #{@options[:ws_user]})") { |u| + @options[:ws_user] = u + } + + opts.separator('') + opts.separator(subtext) + end + + parser.parse!(args) + + if args.length != 1 + puts parser + abort + end + + @options +end + +def invoke_command(commands, component, command) + method = commands[component][command] + if !method.nil? + send(method) + else + puts "Error: unrecognized command '#{command}' for #{component}" + end +end + + + +if $PROGRAM_NAME == __FILE__ + # Bomb out if we're root + if !Gem.win_platform? && Process.uid.zero? + puts "Please run #{@script_name} as a non-root user" + abort + end + + # map component commands to methods + commands = { + database: { + init: :init_db, + reinit: :reinit_db, + delete: :delete_db, + status: :status_db, + start: :start_db, + stop: :stop_db, + restart: :restart_db + }, + webservice: { + init: :init_web_service, + reinit: :reinit_web_service, + delete: :delete_web_service, + status: :status_web_service, + start: :start_web_service, + stop: :stop_web_service, + restart: :restart_web_service + } + } + + parse_args(ARGV) + + command = ARGV[0].to_sym + if @options[:component] == :all + @components.each { |component| + invoke_command(commands, component.to_sym, command) + } + else + invoke_command(commands, @options[:component], command) + end +end