Updated HELK Install & Sysmon Logstash config

- Removed neo4j install (replacing it with something that could scale)
- Added creation of folder /op/helk and cron job in helk_install script
- updated sysmon logstash script to grap intelligence from the new path /opt/helk/otx

logstash/pipeline/11-sysmon-filter.conf

@@ -32,17 +32,17 @@ filter {
       translate {
         field => "[hash][MD5]"
         destination => "[otx][MD5]"
-        dictionary_path => "/opt/otx/otx_md5_.csv"
+        dictionary_path => "/opt/helk/otx/otx_md5_.csv"
       }
       translate {
         field => "[hash][SHA1]"
         destination => "[otx][SHA1]"
-        dictionary_path => "/opt/otx/otx_sha1_.csv"
+        dictionary_path => "/opt/helk/otx/otx_sha1_.csv"
       }
       translate {
         field => "[hash][SHA256]"
         destination => "[otx][SHA256]"
-        dictionary_path => "/opt/otx/otx_sha256_.csv"
+        dictionary_path => "/opt/helk/otx/otx_sha256_.csv"
       }
     }
     if [event_id] == 3 {
@@ -70,7 +70,7 @@ filter {
       translate {
         field => "[destination][ip]"
         destination => "[otx][ip]"
-        dictionary_path => "/opt/otx/otx_ipv4_.csv"
+        dictionary_path => "/opt/helk/otx/otx_ipv4_.csv"
       }
     }
     if [event_id] == 7 {
@@ -98,17 +98,17 @@ filter {
       translate {
         field => "[hash][MD5]"
         destination => "[otx][MD5]"
-        dictionary_path => "/opt/otx/otx_md5_.csv"
+        dictionary_path => "/opt/helk/otx/otx_md5_.csv"
       }
       translate {
         field => "[hash][SHA1]"
         destination => "[otx][SHA1]"
-        dictionary_path => "/opt/otx/otx_sha1_.csv"
+        dictionary_path => "/opt/helk/otx/otx_sha1_.csv"
       }
       translate {
         field => "[hash][SHA256]"
         destination => "[otx][SHA256]"
-        dictionary_path => "/opt/otx/otx_sha256_.csv"
+        dictionary_path => "/opt/helk/otx/otx_sha256_.csv"
       }
     }
     if [event_id] == 8 {

scripts/helk_install.sh

@@ -22,7 +22,6 @@ ERROR=$?
         echoerror "Could not install updates (Error Code: $ERROR)."
     fi
 
-
 echo "[HELK INFO] Installing openjdk-8-jre-headless.."
 apt-get install -y openjdk-8-jre-headless >> $LOGFILE 2>&1
 ERROR=$?
@@ -203,7 +202,7 @@ ERROR=$?
 
 # *********** Installing Pandas ***************
 echo "[HELK INFO] Installing Pandas.."
-pip install pandas>> $LOGFILE 2>&1
+pip install pandas >> $LOGFILE 2>&1
 ERROR=$?
     if [ $ERROR -ne 0 ]; then
         echoerror "Could not install Pandas (Error Code: $ERROR)."
@@ -211,42 +210,22 @@ ERROR=$?
 
 # *********** Copying Intel files to HELK ***************
 echo "[HELK INFO] Copying Intel files to HELK"
-mkdir /opt/otx
+mkdir /opt/helk/
-cp -v ../logstash/intel/* /opt/otx/>> $LOGFILE 2>&1
+mkdir /opt/helk/otx
+cp -v ../logstash/intel/* /opt/helk/otx/ >> $LOGFILE 2>&1
 ERROR=$?
     if [ $ERROR -ne 0 ]; then
         echoerror "Could not copy intel files to HELK (Error Code: $ERROR)."
     fi
-
+# *********** Creating Cron Job to run OTX script every monday at 8AM and capture last 30 days of Intel *************
-# *********** Download Neo4j public signing key **********.
+echo "[HELK INFO] Creating a cronjob for OTX intel script"
-echo "[HELK INFO] Downloading Neo4j public signing key and adding it to the host.."
+mkdir /opt/helk/scripts
-wget -O - https://debian.neo4j.org/neotechnology.gpg.key | sudo apt-key add - >> $LOGFILE 2>&1
+cp -v otx_helk.py /opt/helk/scripts/
+cronjob="0 8 * * 1 python /opt/helk/scripts/otx_helk.py"
+(crontab -u root -l; echo "$cronjob" ) | crontab -u root - >> $LOGFILE 2>&1
 ERROR=$?
     if [ $ERROR -ne 0 ]; then
-        echoerror "Could not download Neo4j public signing key and add it to the host (Error Code: $ERROR)."
+        echoerror "Could not create cronjob for OTX intel script (Error Code: $ERROR)."
-    fi
-
-# *********** Upgrade repository sources **********.
-echo "[HELK INFO] Upgrading repository sources.."
-echo 'deb https://debian.neo4j.org/repo stable/' | sudo tee /etc/apt/sources.list.d/neo4j.list >> $LOGFILE 2>&1
-ERROR=$?
-    if [ $ERROR -ne 0 ]; then
-        echoerror "Could not upgrade repository sources (Error Code: $ERROR)."
-    fi
-
-echo "[HELK INFO] Installing updates.."
-apt-get update >> $LOGFILE 2>&1
-ERROR=$?
-    if [ $ERROR -ne 0 ]; then
-        echoerror "Could not install update (Error Code: $ERROR)."
-    fi
-
-# *********** Install Neo4j **********.
-echo "[HELK INFO] Installing Neo4j.."
-apt-get -y install neo4j >> $LOGFILE 2>&1
-ERROR=$?
-    if [ $ERROR -ne 0 ]; then
-        echoerror "Could not install neo4j (Error Code: $ERROR)."
     fi
 
 # *********** Installing Logstash ***************