05312018

Logstash
- Added Local Pipeline to the build to allow custom local configurations
- updated sysmon config to fix https://github.com/Cyb3rWard0g/HELK/issues/63
- removed port exposed in local logstash Dockerfile. It will be pushed to official docker image in the next update
- removed logstash init file (not being used anymore)

Zeppelin
- not available yet
- initial draft dockerfile
- created spark-defaults file for future zeppelin dockerfile

Install Script
- incrased minimum memory size required

docker-compose.yml

@@ -20,6 +20,8 @@ services:
   helk-logstash:
     image: cyb3rward0g/helk-logstash:6.2.4
     container_name: helk-logstash
+    volumes:
+      - ./helk-logstash/pipeline:/usr/share/logstash/pipeline
     environment:
       - "LS_JAVA_OPTS=-Xms2g -Xmx2g"
     restart: always

helk-logstash/Dockerfile

@@ -16,5 +16,3 @@ LABEL description="Dockerfile base for the HELK Logstash."
 ADD logstash.yml /usr/share/logstash/config/logstash.yml
 ADD pipeline /usr/share/logstash/pipeline
 ADD output_templates /usr/share/logstash/output_templates
-
-EXPOSE 5044

helk-logstash/logstash-init

@@ -1,202 +0,0 @@
-#!/bin/sh
-# Init script for logstash
-# Maintained by Roberto Rodriguez @Cyb3rWard0g
-# Reference: 
-# https://github.com/elastic/logstash/blob/master/distribution/rpm/src/main/packaging/init.d/logstash
-# https://github.com/spujadas/elk-docker/blob/master/logstash-init
-
-### BEGIN INIT INFO
-# Provides:          logstash
-# Required-Start:    $remote_fs $syslog
-# Required-Stop:     $remote_fs $syslog
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description:
-# Description:        Starts Logstash as a daemon.
-### END INIT INFO
-
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-NAME=logstash
-DEFAULT=/etc/default/$NAME
-export PATH
-
-if [ $(id -u) -ne 0 ]; then
-   echo "You need root privileges to run this script"
-   exit 1
-fi
-
-. /lib/lsb/init-functions
-
-if [ -r /etc/default/rcS ]; then
-	. /etc/default/rcS
-fi
-
-# The following variables can be overwritten in $DEFAULT
-
-JAVACMD=/usr/bin/java
-LS_HOME=/usr/share/logstash
-LS_SETTINGS_DIR=/etc/logstash
-LS_CONF_PATH=/etc/logstash/pipeline
-LS_LOGS_PATH=/var/log/logstash
-LS_LOGS_FILE=${LS_LOGS_PATH}/${NAME}-plain.log
-#LS_JAVA_OPTS=""
-LS_PIDFILE=/var/run/logstash.pid
-LS_USER=logstash
-LS_GROUP=logstash
-LS_GC_LOG_FILE=/var/log/logstash/gc.log
-LS_OPEN_FILES=16384
-LS_NICE=19
-SERVICE_NAME="logstash"
-SERVICE_DESCRIPTION="logstash"
-
-# End of variables that can be overwritten in $DEFAULT
-
-# overwrite settings from default file
-if [ -f "$DEFAULT" ]; then
-	. "$DEFAULT"
-fi
-
-# Define other required variables
-LS_EXEC=$LS_HOME/bin/logstash
-#LS_EXEC_OPTS="--path.settings ${LS_SETTINGS_DIR} --path.config ${LS_CONF_PATH} --path.logs ${LS_LOGS_PATH}"
-LS_EXEC_OPTS="--path.settings ${LS_SETTINGS_DIR}"
-
-export LS_JAVA_OPTS
-export LS_HOME
-export LS_SETTINGS_DIR
-
-if [ ! -x "$LS_EXEC" ]; then
-	echo "The logstash startup script does not exists or it is not executable, tried: $LS_EXEC"
-	exit 1
-fi
-
-touch ${LS_LOGS_FILE}
-chown ${LS_USER}:${LS_GROUP} ${LS_LOGS_FILE}
-
-checkJava() {
-    if [ -x "$JAVACMD" ]; then
-        JAVA="$JAVACMD"
-    else
-        JAVA=`which java`
-    fi
-
-    if [ ! -x "$JAVA" ]; then
-        echo "Could not find any executable java binary. Please install java in your PATH or set JAVACMD"
-        exit 1
-    fi
-}
-
-start() {
-	checkJava
-
-	echo "Starting $NAME"
-
-	if [ -n "$LS_PIDFILE" ] && [ ! -e "$LS_PIDFILE" ]; then
-		touch "$LS_PIDFILE" && chown logstash:logstash "$LS_PIDFILE"
-	fi
-
-	if [ -n "$LS_OPEN_FILES" ]; then
-		ulimit -n $LS_OPEN_FILES
-	fi
-
-	# Start Service
-	nice -n$LS_NICE chroot --userspec $LS_USER:$LS_GROUP / sh -c "
-        cd $LS_HOME
-        ulimit -n ${LS_OPEN_FILES}
-        exec $LS_EXEC $LS_EXEC_OPTS
-    " &
-
-  # Generate the pidfile from here. If we instead made the forked process
-  # generate it there will be a race condition between the pidfile writing
-  # and a process possibly asking for status.
-  echo $! > $LS_PIDFILE
-
-  echo "$NAME started."
-  return 0
-}
-
-stop() {
-  # Try a few times to kill TERM the program
-  if status; then
-    pid=$(cat "$LS_PIDFILE")
-    echo "Killing $NAME (pid $pid) with SIGTERM"
-    kill -TERM $pid
-    # Wait for it to exit.
-    for i in 1 2 3 4 5; do
-      echo "Waiting for $NAME (pid $pid) to die..."
-      status || break
-      sleep 1
-    done
-    if status; then
-      echo "$NAME stop failed; still running."
-    else
-      echo "$NAME stopped."
-      rm -f $LS_PIDFILE
-    fi
-  fi
-}
-
-status() {
-  if [ -f "$LS_PIDFILE" ] ; then
-    pid=$(cat "$LS_PIDFILE")
-    if kill -0 $pid > /dev/null 2> /dev/null; then
-      # process by this pid is running.
-      # It may not be our pid, but that's what you get with just pidfiles.
-      # TODO(sissel): Check if this process seems to be the same as the one we
-      # expect. It'd be nice to use flock here, but flock uses fork, not exec,
-      # so it makes it quite awkward to use in this case.
-      return 0
-    else
-      return 2 # program is dead but pid file exists
-    fi
-  else
-    return 3 # program is not running
-  fi
-}
-
-force_stop() {
-  if status; then
-    stop
-    status && kill -KILL $(cat "$LS_PIDFILE")
-    rm -f $LS_PIDFILE
-  fi
-}
-
-case "$1" in
-  start)
-    status
-    code=$?
-    if [ $code -eq 0 ]; then
-      echo "$NAME is already running"
-    else
-      start
-      code=$?
-    fi
-    exit $code
-    ;;
-
-  stop) stop ;;
-
-  force-stop) force_stop ;;
-
-  status)
-    status
-    code=$?
-    if [ $code -eq 0 ]; then
-      echo "$NAME is running"
-    else
-      echo "$NAME is not running"
-    fi
-    exit $code
-    ;;
-
-  restart) stop && start ;;
-
-  *)
-    echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
-    exit 3
-    ;;
-esac
-
-exit $?
-

helk-logstash/pipeline/11-winevent-sysmon-filter.conf

@@ -298,7 +298,8 @@ filter {
       remove_field => "[event_data][UtcTime]"
       remove_field => "[event_data][CreationUtcTime]"
       remove_field => "[event_data][PreviousCreationUtcTime]"
-      rename => { "[event_data][User]" => "user"}
+      remove_field => "[user]"
+      rename => { "[event_data][User]" => "user_account"}
     }
   }
 }

helk-zeppelin/Dockerfile

@@ -10,86 +10,80 @@ LABEL description="Dockerfile base for the HELK Zeppelin."
 
 ENV DEBIAN_FRONTEND noninteractive
 
+USER root
 # *********** Installing Prerequisites ***************
-# `Z_VERSION` will be updated by `dev/change_zeppelin_version.sh`
+ENV Z_VERSION="0.8."
-ENV Z_VERSION="0.7.3"
 ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
-    Z_HOME="/zeppelin" \
+    Z_HOME="/zeppelin"
-    LANG=en_US.UTF-8 \
-    LC_ALL=en_US.UTF-8
 
-RUN echo "$LOG_TAG update and install basic packages" && \
+ENV Z_GID=710
-    apt-get -y update && \
+ENV Z_UID=710
-    apt-get install -y locales && \
+ENV Z_USER=zelk
-    locale-gen $LANG && \
-    apt-get install -y software-properties-common && \
-    apt -y autoclean && \
-    apt -y dist-upgrade && \
-    apt-get install -y build-essential
 
 ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
 RUN echo "$LOG_TAG Install java8" && \
     apt-get -y update && \
-    apt-get install -y openjdk-8-jdk && \
+    apt-get install -y git openjdk-8-jdk \
+    libfontconfig git build-essential chrpath \
+    libssl-dev libxft-dev libfreetype6 libfreetype6-dev \
+    libfontconfig1 libfontconfig1-dev python3-pip && \
     rm -rf /var/lib/apt/lists/*
 
-# should install conda first before numpy, matploylib since pip and python will be installed by conda
+# *********** Upgrading PIP ***************
-RUN echo "$LOG_TAG Install miniconda2 related packages" && \
+RUN pip3 install --upgrade pip
-    apt-get -y update && \
+
-    apt-get install -y bzip2 ca-certificates \
+# *********** Create User *****************
-    libglib2.0-0 libxext6 libsm6 libxrender1 \
+RUN groupadd -r zelk -g ${Z_GID} \
-    git mercurial subversion && \
+  && useradd -m -s /bin/bash -N -u $Z_UID $Z_USER \
-    echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
+  && chmod g+w /etc/passwd /etc/group \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \
+  && chown -R zelk:zelk /usr/local/ /tmp /usr/bin ${SPARK_HOME}
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh
-ENV PATH /opt/conda/bin:$PATH
 
 RUN echo "$LOG_TAG Install python related packages" && \
     apt-get -y update && \
-    apt-get install -y python-dev python-pip && \
     apt-get install -y gfortran && \
-    # numerical/algebra packages
+   # numerical/algebra packages
     apt-get install -y libblas-dev libatlas-dev liblapack-dev && \
     # font, image for matplotlib
     apt-get install -y libpng-dev libfreetype6-dev libxft-dev && \
     # for tkinter
     apt-get install -y python-tk libxml2-dev libxslt-dev zlib1g-dev && \
-    pip install numpy && \
+    pip3 install numpy && \
-    pip install matplotlib
+    pip3 install matplotlib
 
-RUN echo "$LOG_TAG Install R related packages" && \
+# ************** Install PhantpmJS ****************
-    echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list && \
+USER $Z_UID
-    gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 && \
-    gpg -a --export E084DAB9 | apt-key add - && \
-    apt-get -y update && \
-    apt-get -y install r-base r-base-dev && \
-    R -e "install.packages('knitr', repos='http://cran.us.r-project.org')" && \
-    R -e "install.packages('ggplot2', repos='http://cran.us.r-project.org')" && \
-    R -e "install.packages('googleVis', repos='http://cran.us.r-project.org')" && \
-    R -e "install.packages('data.table', repos='http://cran.us.r-project.org')" && \
-    # for devtools, Rcpp
-    apt-get -y install libcurl4-gnutls-dev libssl-dev && \
-    R -e "install.packages('devtools', repos='http://cran.us.r-project.org')" && \
-    R -e "install.packages('Rcpp', repos='http://cran.us.r-project.org')" && \
-    Rscript -e "library('devtools'); library('Rcpp'); install_github('ramnathv/rCharts')"
 
 # ************** Install Maven *********************
 ENV MAVEN_VERSION 3.5.3
-RUN wget wget -qO- http://www.us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | sudo tar xvz -C /usr/local && \
+RUN wget -qO- http://www.us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | tar xvz -C /usr/local && \
     ln -s /usr/local/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn
 
-RUN echo "$LOG_TAG Download Zeppelin binary" && \
+USER root
-    wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
+# ************** Install Zeppelin *********************
-    tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
+RUN echo "$LOG_TAG Download Zeppelin" && \
-    rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
+    cd /tmp && git clone --branch v0.8.0-rc2 https://github.com/apache/zeppelin.git && \
-    mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
+    mv /tmp/zeppelin ${Z_HOME}
+
+RUN chown -R zelk:zelk ${Z_HOME}
+
+USER $Z_UID
+
+RUN cd $Z_HOME && \
+    mvn clean package -DskipTests -X
+
+# *********** Install CAPS ***************
+RUN cd ${Z_HOME} && \
+    wget https://github.com/opencypher/cypher-for-apache-spark/releases/download/1.0.0-beta7/spark-cypher-1.0.0-beta7-cluster.jar
+
+ADD spark-defaults.conf ${SPARK_HOME}/conf/
+
+USER root
  
 RUN echo "$LOG_TAG Cleanup" && \
     apt-get autoclean && \
     apt-get clean
 
-EXPOSE 8080
+EXPOSE $ZEPPELIN_PORT
 
 WORKDIR ${Z_HOME}
 CMD ["bin/zeppelin.sh"]

helk-zeppelin/spark-defaults.conf

@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Default system properties included when running spark-submit.
+# This is useful for setting default environmental settings.
+
+# Example:
+# spark.master                     spark://master:7077
+# spark.eventLog.enabled           true
+# spark.eventLog.dir               hdfs://namenode:8021/directory
+# spark.serializer                 org.apache.spark.serializer.KryoSerializer
+# spark.driver.memory              5g
+# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
+
+# HELK References:
+# https://spark.apache.org/docs/latest/configuration.html
+# https://graphframes.github.io/quick-start.html
+# https://spark-packages.org/package/graphframes/graphframes
+# https://spark.apache.org/docs/latest/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow
+
+#spark.master spark://helk-spark-master:7077
+spark.jars /zeppelin/spark-cypher-1.0.0-beta7-cluster.jar
+spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3
+spark.sql.execution.arrow.enabled true

helk-zeppelin/zeppelin-env.sh.template

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# export JAVA_HOME=
+# export MASTER=                 		# Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
+# export ZEPPELIN_JAVA_OPTS      		# Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
+# export ZEPPELIN_MEM            		# Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
+# export ZEPPELIN_INTP_MEM       		# zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
+# export ZEPPELIN_INTP_JAVA_OPTS 		# zeppelin interpreter process jvm options.
+# export ZEPPELIN_SSL_PORT       		# ssl port (used when ssl environment variable is set to true)
+
+# export ZEPPELIN_LOG_DIR        		# Where log files are stored.  PWD by default.
+# export ZEPPELIN_PID_DIR        		# The pid files are stored. ${ZEPPELIN_HOME}/run by default.
+# export ZEPPELIN_WAR_TEMPDIR    		# The location of jetty temporary directory.
+# export ZEPPELIN_NOTEBOOK_DIR   		# Where notebook saved
+# export ZEPPELIN_NOTEBOOK_HOMESCREEN		# Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
+# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE	# hide homescreen notebook from list when this value set to "true". default "false"
+# export ZEPPELIN_NOTEBOOK_S3_BUCKET        # Bucket where notebook saved
+# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT      # Endpoint of the bucket
+# export ZEPPELIN_NOTEBOOK_S3_USER          # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
+# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID    # AWS KMS key ID
+# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION      # AWS KMS key region
+# export ZEPPELIN_IDENT_STRING   		# A string representing this instance of zeppelin. $USER by default.
+# export ZEPPELIN_NICENESS       		# The scheduling priority for daemons. Defaults to 0.
+# export ZEPPELIN_INTERPRETER_LOCALREPO         # Local repository for interpreter's additional dependency loading
+# export ZEPPELIN_INTERPRETER_DEP_MVNREPO       # Remote principal repository for interpreter's additional dependency loading
+# export ZEPPELIN_HELIUM_NPM_REGISTRY           # Remote Npm registry for Helium dependency loader
+# export ZEPPELIN_NOTEBOOK_STORAGE 		# Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
+# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC	# If there are multiple notebook storages, should we treat the first one as the only source of truth?
+# export ZEPPELIN_NOTEBOOK_PUBLIC   # Make notebook public by default when created, private otherwise
+
+#### Spark interpreter configuration ####
+
+## Use provided spark installation ##
+## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit
+##
+# export SPARK_HOME                             # (required) When it is defined, load it instead of Zeppelin embedded Spark libraries
+# export SPARK_SUBMIT_OPTIONS                   # (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G".
+# export SPARK_APP_NAME                         # (optional) The name of spark application.
+
+## Use embedded spark binaries ##
+## without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries.
+## however, it is not encouraged when you can define SPARK_HOME
+##
+# Options read in YARN client mode
+# export HADOOP_CONF_DIR         		# yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR.
+# Pyspark (supported with Spark 1.2.1 and above)
+# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
+# export PYSPARK_PYTHON          		# path to the python command. must be the same path on the driver(Zeppelin) and all workers.
+# export PYTHONPATH
+
+## Spark interpreter options ##
+##
+# export ZEPPELIN_SPARK_USEHIVECONTEXT  # Use HiveContext instead of SQLContext if set true. true by default.
+# export ZEPPELIN_SPARK_CONCURRENTSQL   # Execute multiple SQL concurrently if set true. false by default.
+# export ZEPPELIN_SPARK_IMPORTIMPLICIT  # Import implicits, UDF collection, and sql if set true. true by default.
+# export ZEPPELIN_SPARK_MAXRESULT       # Max number of Spark SQL result to display. 1000 by default.
+# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE       # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
+
+
+#### HBase interpreter configuration ####
+
+## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
+
+# export HBASE_HOME=                    # (require) Under which HBase scripts and configuration should be
+# export HBASE_CONF_DIR=                # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml
+
+#### ZeppelinHub connection configuration ####
+# export ZEPPELINHUB_API_ADDRESS		# Refers to the address of the ZeppelinHub service in use
+# export ZEPPELINHUB_API_TOKEN			# Refers to the Zeppelin instance token of the user
+# export ZEPPELINHUB_USER_KEY			# Optional, when using Zeppelin with authentication.
+
+#### Zeppelin impersonation configuration
+# export ZEPPELIN_IMPERSONATE_CMD       # Optional, when user want to run interpreter as end web user. eg) 'sudo -H -u ${ZEPPELIN_IMPERSONATE_USER} bash -c '
+# export ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER  #Optional, by default is true; can be set to false if you don't want to use --proxy-user option with Spark interpreter when impersonation enabled

helk_install.sh

@@ -29,7 +29,7 @@ check_min_requirements(){
         ES_MEMORY=$(free -hm | awk 'NR==2{printf "%.f", $4/2 }')
         AVAILABLE_DISK=$(df -h | awk '$NF=="/"{printf "%.f\t\t", $4}')
         
-        if [ "${AVAILABLE_MEMORY}" -ge "10" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then
+        if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then
             echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY"
             echo "[HELK-INSTALLATION-INFO] Available Disk: $AVAILABLE_DISK"
         else