Land #9252, docker improvements

.dockerignore

@@ -34,7 +34,7 @@ config/database.yml
 # target config file for testing
 features/support/targets.yml
 # simplecov coverage data
-coverage
+coverage/
 doc/
 external/source/meterpreter/java/bin
 external/source/meterpreter/java/build

.gitignore

@@ -88,6 +88,7 @@ data/meterpreter/ext_server_pivot.*.dll
 
 # local docker compose overrides
 docker-compose.local*
+.env
 
 # Ignore python bytecode
 *.pyc

.travis.yml

@@ -25,7 +25,7 @@ matrix:
 jobs:
   # build docker image
   include:
-  - env: CMD="docker-compose -f $TRAVIS_BUILD_DIR/docker-compose.yml build" DOCKER="true"
+  - env: CMD="docker-compose build" DOCKER="true"
     # we do not need any setup
     before_install: skip
     install: skip

Dockerfile

@@ -1,14 +1,17 @@
 FROM ruby:2.4.2-alpine
-MAINTAINER Rapid7
+LABEL maintainer="Rapid7"
 
 ARG BUNDLER_ARGS="--jobs=8 --without development test coverage"
 ENV APP_HOME /usr/src/metasploit-framework/
 ENV MSF_USER msf
 ENV NMAP_PRIVILEGED=""
+ENV BUNDLE_IGNORE_MESSAGES="true"
 WORKDIR $APP_HOME
 
-COPY Gemfile* m* Rakefile $APP_HOME
-COPY lib $APP_HOME/lib
+COPY Gemfile* metasploit-framework.gemspec Rakefile $APP_HOME
+COPY lib/metasploit/framework/version.rb $APP_HOME/lib/metasploit/framework/version.rb
+COPY lib/metasploit/framework/rails_version_constraint.rb $APP_HOME/lib/metasploit/framework/rails_version_constraint.rb
+COPY lib/msf/util/helper.rb $APP_HOME/lib/msf/util/helper.rb
 
 RUN apk update && \
     apk add \
@@ -36,8 +39,7 @@ RUN apk update && \
       ncurses-dev \
       git \
     && echo "gem: --no-ri --no-rdoc" > /etc/gemrc \
-    # this currently fails: https://github.com/rubygems/rubygems/issues/2064
-    # && gem update --system \
+    && gem update --system \
     && gem install bundler \
     && bundle install --system $BUNDLER_ARGS \
     && apk del .ruby-builddeps \
@@ -46,7 +48,7 @@ RUN apk update && \
 RUN adduser -g msfconsole -D $MSF_USER
 
 RUN /usr/sbin/setcap cap_net_raw,cap_net_bind_service=+eip $(which ruby)
-RUN /usr/sbin/setcap cap_net_raw,cap_net_bind_service=+eip /usr/bin/nmap
+RUN /usr/sbin/setcap cap_net_raw,cap_net_bind_service=+eip $(which nmap)
 
 USER $MSF_USER
 

docker-compose.override.yml

@@ -1,13 +1,14 @@
-version: '2'
+version: '3'
 
 services:
   ms:
     build:
+      context: .
+      dockerfile: ./Dockerfile
       args:
         BUNDLER_ARGS: --jobs=8
     image: metasploit:dev
     environment:
       DATABASE_URL: postgres://postgres@db:5432/msf_dev
-
     volumes:
       - .:/usr/src/metasploit-framework

docker-compose.yml

@@ -1,10 +1,7 @@
-version: '2'
+version: '3'
 services:
   ms:
-    image: metasploit
-    build:
-      context: .
-      dockerfile: ./Dockerfile
+    image: metasploitframework/metasploit-framework:latest
     environment:
       DATABASE_URL: postgres://postgres@db:5432/msf
     links:
@@ -16,7 +13,7 @@ services:
       - /etc/localtime:/etc/localtime:ro
 
   db:
-    image: postgres:9-alpine
+    image: postgres:10-alpine
     volumes:
       - pg_data:/var/lib/postgresql/data
 

docker/README.md

@@ -3,43 +3,36 @@
 
 To run `msfconsole`
 ```bash
+docker-compose build
 docker-compose run --rm --service-ports ms
 ```
+or
+```bash
+./docker/bin/msfconsole
+```
 
 To run `msfvenom`
 ```bash
-docker-compose run --rm ms ./msfvenom
+docker-compose build
+docker-compose run --rm --no-deps ms ./msfvenom
 ```
-
-### I don't like typing `docker-compose --rm ...`
-
-We have included some binstubs `./bin`, you can symlink them to your path.
-
-Assuming you have `$HOME/bin`, and it's in your `$PATH`. You can run this from the project root:
-
+or
 ```bash
-ln -s `pwd`/docker/bin/msfconsole $HOME/bin/
-ln -s `pwd`/docker/bin/msfvenom $HOME/bin/
+./docker/bin/msfvenom
 ```
 
-If you set the environment variable `MSF_BUILD` the container will be rebuilt.
-
-```bash
-MSF_BUILD=1 ./docker/bin/msfconsole
-MSF_BUILD=1 ./docker/bin/msfconsole-dev
-```
+You can pass any command line arguments to the binstubs or the docker-compose command and they will be passed to `msfconsole` or `msfvenom`. If you need to rebuild an image (for example when the Gemfile changes) you need to build the docker image using `docker-compose build` or supply the `--rebuild` parameter to the binstubs.
 
 ### But I want reverse shells...
 
-By default we expose port `4444`. You'll need to set `LHOST` to be a hostname/ip
-of your host machine.
+By default we expose port `4444`.
 
 If you want to expose more ports, or have `LHOST` prepopulated with a specific
 value; you'll need to setup a local docker-compose override for this.
 
-Create `docker/docker-compose.local.override.yml` with:
+Create `docker-compose.local.override.yml` with:
 ```yml
-version: '2'
+version: '3'
 services:
   ms:
     environment:
@@ -56,19 +49,6 @@ Now you need to set the `COMPOSE_FILE` environment variable to load your local
 override.
 
 ```bash
-echo "COMPOSE_FILE=./docker-compose.yml:./docker/docker-compose.local.override.yml" >> .env
+echo "COMPOSE_FILE=./docker-compose.yml:./docker-compose.override.yml:./docker-compose.local.override.yml" >> .env
 ```
 Now you should be able get reverse shells working
-
-## Developing
-
-To setup you environment for development, you need to add `docker/docker-compose.development.override.yml`
-to your `COMPOSE_FILE` environment variable.
-
-If you don't have a `COMPOSE_FILE` environment variable, you can set it up with this:
-
-```bash
-echo "COMPOSE_FILE=./docker-compose.yml:./docker/docker-compose.development.override.yml" >> .env
-```
-
-Alternatively you can also use the `msfconsole-dev` binstub.

docker/bin/msfconsole

@@ -19,8 +19,12 @@ fi
 
 cd $MSF_PATH
 
-if [[ -n "$MSF_BUILD" ]]; then
-  docker-compose -f $MSF_PATH/docker-compose.yml build
+PARAMS="$@"
+
+if [[ $PARAMS == *"--rebuild"* ]]; then
+  echo "Rebuilding image"
+  docker-compose build
+  exit $?
 fi
 
-docker-compose run --rm --service-ports ms ./msfconsole -r docker/msfconsole.rc "$@"
+docker-compose run --rm --service-ports ms ./msfconsole -r docker/msfconsole.rc "$PARAMS"

docker/bin/msfconsole-dev

@@ -1,27 +0,0 @@
-#! /bin/bash
-
-if [[ -z "$MSF_PATH" ]]; then
-  path=`dirname $0`
-
-  # check for ./docker/msfconsole.rc
-  if [[ ! -f $path/../msfconsole.rc ]] ; then
-
-    # we are not inside the project
-    realpath --version > /dev/null 2>&1 || { echo >&2 "I couldn't find where metasploit is. Set \$MSF_PATH or execute this from the project root"; exit 1 ;}
-
-    # determine script path
-    pushd $(dirname $(realpath $0)) > /dev/null
-    path=$(pwd)
-    popd > /dev/null
-  fi
-  MSF_PATH=$(dirname $(dirname $path))
-fi
-
-cd $MSF_PATH
-
-if [[ -n "$MSF_BUILD" ]]; then
-  docker-compose -f $MSF_PATH/docker-compose.yml -f $MSF_PATH/docker/docker-compose.development.override.yml build
-fi
-
-docker-compose -f $MSF_PATH/docker-compose.yml -f $MSF_PATH/docker/docker-compose.development.override.yml run --rm --service-ports ms ./msfconsole -r docker/msfconsole.rc "$@"
-

docker/bin/msfvenom

@@ -17,9 +17,15 @@ if [[ -z "$MSF_PATH" ]]; then
   MSF_PATH=$(dirname $(dirname $path))
 fi
 
-if [[ -n "$MSF_BUILD" ]]; then
-  docker-compose -f $MSF_PATH/docker-compose.yml build
+cd $MSF_PATH
+
+PARAMS="$@"
+
+if [[ $PARAMS == *"--rebuild"* ]]; then
+  echo "Rebuilding image"
+  docker-compose build
+  exit $?
 fi
 
-cd $MSF_PATH
-docker-compose run --rm --service-ports ms ./msfvenom "$@"
+# we need no database here
+docker-compose run --rm --no-deps ms ./msfvenom "$PARAMS"

docker/bin/msfvenom-dev

@@ -1,26 +0,0 @@
-#! /bin/bash
-
-if [[ -z "$MSF_PATH" ]]; then
-  path=`dirname $0`
-
-  # check for ./docker/msfconsole.rc
-  if [[ ! -f $path/../msfconsole.rc ]] ; then
-
-    # we are not inside the project
-    realpath --version > /dev/null 2>&1 || { echo >&2 "I couldn't find where metasploit is. Set \$MSF_PATH or execute this from the project root"; exit 1 ;}
-
-    # determine script path
-    pushd $(dirname $(realpath $0)) > /dev/null
-    path=$(pwd)
-    popd > /dev/null
-  fi
-  MSF_PATH=$(dirname $(dirname $path))
-fi
-
-cd $MSF_PATH
-
-if [[ -n "$MSF_BUILD" ]]; then
-  docker-compose -f $MSF_PATH/docker-compose.yml -f $MSF_PATH/docker/docker-compose.development.override.yml build
-fi
-
-docker-compose -f $MSF_PATH/docker-compose.yml -f $MSF_PATH/docker/docker-compose.development.override.yml run --rm --service-ports ms ./msfvenom "$@"

lib/msf/ui/console/command_dispatcher/db.rb

@@ -1826,6 +1826,8 @@ class Db
     if (path)
       auth, dest = path.split('@')
       (dest = auth and auth = nil) if not dest
+      # remove optional scheme in database url
+      auth = auth.sub(/^\w+:\/\//, "") if auth
       res[:user],res[:pass] = auth.split(':') if auth
       targ,name = dest.split('/')
       (name = targ and targ = nil) if not name