From 3178c851724bac88e76fe2d76cf3954190e2eef9 Mon Sep 17 00:00:00 2001 From: Roberto Rodriguez Date: Wed, 20 Dec 2017 14:55:57 -0500 Subject: [PATCH] Updated scripts, Logstash confs, elasticsearch conf & created sysmon template - Logstash -- Cleaned output configurations -- Created Sysmon teamplte -- Added sysmon template to sysmon elasticsearch output -- Removed sniffing = True from every elasticsearch output - Scripts -- Updated Install config -- Added creation of Kibana index patterns to install script -- Added headers to every script but posh script -- renamed scripts to keep naming standard helk-* --- elasticsearch/elasticsearch.yml | 6 +- .../output_templates/sysmon-template.json | 92 +++++++++++++++++++ logstash/pipeline/10-powershell-filter.conf | 6 ++ logstash/pipeline/11-sysmon-filter.conf | 6 ++ logstash/pipeline/50-sysmon-output.conf | 5 +- .../pipeline/51-winevent-security-output.conf | 9 +- .../pipeline/52-winevent-system-output.conf | 7 +- .../53-winevent-application-output.conf | 7 +- logstash/pipeline/54-powershell-output.conf | 7 +- scripts/helk_docker_install.sh | 10 +- ...date_geoipdb.sh => helk_geoipdb_update.sh} | 16 +++- scripts/helk_install.sh | 41 +++++++-- scripts/helk_kibana_index_pattern_creation.sh | 34 +++++++ scripts/{otx_helk.py => helk_otx.py} | 8 ++ 14 files changed, 226 insertions(+), 28 deletions(-) create mode 100644 logstash/output_templates/sysmon-template.json rename scripts/{update_geoipdb.sh => helk_geoipdb_update.sh} (67%) create mode 100755 scripts/helk_kibana_index_pattern_creation.sh rename scripts/{otx_helk.py => helk_otx.py} (91%) mode change 100644 => 100755 diff --git a/elasticsearch/elasticsearch.yml b/elasticsearch/elasticsearch.yml index 71c289e..85eaddb 100644 --- a/elasticsearch/elasticsearch.yml +++ b/elasticsearch/elasticsearch.yml @@ -30,11 +30,11 @@ # # Path to directory where to store the data (separate multiple locations by comma): # -#path.data: /path/to/data +path.data: /var/lib/elasticsearch # # Path to log files: # -#path.logs: /path/to/logs +path.logs: /var/log/elasticsearch # # ----------------------------------- Memory ----------------------------------- # @@ -69,7 +69,7 @@ network.host: localhost # # Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): # -#discovery.zen.minimum_master_nodes: 3 +#discovery.zen.minimum_master_nodes: # # For more information, consult the zen discovery module documentation. # diff --git a/logstash/output_templates/sysmon-template.json b/logstash/output_templates/sysmon-template.json new file mode 100644 index 0000000..c749374 --- /dev/null +++ b/logstash/output_templates/sysmon-template.json @@ -0,0 +1,92 @@ +{ + "template" : "sysmon-*", + "settings" : { + "index.refresh_interval": "5s" + }, + "mappings":{ + "doc":{ + "dynamic_templates": [{ + "strings":{ + "match_mapping_type": "string", + "mapping": { + "type": "text", + "norms": false, + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }], + "properties":{ + "@timestamp":{"type":"date"}, + "destination":{ + "properties":{ + "port":{ + "properties":{ + "number":{"type":"integer"} + } + } + } + }, + "event_data":{"type":"object"}, + "event_id":{"type":"integer"}, + "geoip":{ + "properties":{ + "dma_code":{"type":"integer"}, + "latitude":{"type":"float"}, + "location":{"type": "geo_point"}, + "longitude":{"type":"float"}, + "postal_code":{"type":"integer"} + } + }, + "network":{ + "properties":{ + "initiated":{"type":"boolean"} + } + }, + "process":{ + "properties":{ + "id":{"type":"integer"}, + "image":{ + "properties":{ + "signed":{"type":"boolean"} + } + }, + "parent":{ + "properties":{ + "id":{"type":"integer"} + } + }, + "source":{ + "properties":{ + "id":{"type":"integer"}, + "threadid":{"type":"integer"} + } + }, + "target":{ + "properties":{ + "id":{"type":"integer"} + } + }, + "terminalsessionid":{"type":"integer"} + } + }, + "process_id":{"type":"integer"}, + "source":{ + "properties":{ + "port":{ + "properties":{ + "number":{"type":"integer"} + } + } + } + }, + "thread_id":{"type":"integer"}, + "version":{"type":"integer"} + } + } + } +} diff --git a/logstash/pipeline/10-powershell-filter.conf b/logstash/pipeline/10-powershell-filter.conf index 3b247f6..b7f6624 100644 --- a/logstash/pipeline/10-powershell-filter.conf +++ b/logstash/pipeline/10-powershell-filter.conf @@ -1,3 +1,9 @@ +# HELK filter conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + filter { if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{ if [event_id] == 4103 { diff --git a/logstash/pipeline/11-sysmon-filter.conf b/logstash/pipeline/11-sysmon-filter.conf index e2a3ca4..972eee8 100644 --- a/logstash/pipeline/11-sysmon-filter.conf +++ b/logstash/pipeline/11-sysmon-filter.conf @@ -1,3 +1,9 @@ +# HELK filter conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + filter { if [log_name] == "Microsoft-Windows-Sysmon/Operational"{ if [event_id] == 1 { diff --git a/logstash/pipeline/50-sysmon-output.conf b/logstash/pipeline/50-sysmon-output.conf index 7ab0a24..077e083 100644 --- a/logstash/pipeline/50-sysmon-output.conf +++ b/logstash/pipeline/50-sysmon-output.conf @@ -2,9 +2,10 @@ output { if [log_name] == "Microsoft-Windows-Sysmon/Operational"{ elasticsearch { hosts => ["elasticsearch:9200", "127.0.0.1:9200"] - sniffing => true - manage_template => false index => "sysmon-%{+YYYY.MM.dd}" + template => "/opt/helk/templates/sysmon-template.json" + template_name => "sysmon" + template_overwrite => true } } } \ No newline at end of file diff --git a/logstash/pipeline/51-winevent-security-output.conf b/logstash/pipeline/51-winevent-security-output.conf index a3d8d4c..24b833d 100644 --- a/logstash/pipeline/51-winevent-security-output.conf +++ b/logstash/pipeline/51-winevent-security-output.conf @@ -1,10 +1,15 @@ +# HELK output conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + output { if [log_name] == "Security"{ elasticsearch { hosts => ["elasticsearch:9200", "127.0.0.1:9200"] - sniffing => true manage_template => false - index => "winevent-security--%{+YYYY.MM.dd}" + index => "winevent-security-%{+YYYY.MM.dd}" } } } \ No newline at end of file diff --git a/logstash/pipeline/52-winevent-system-output.conf b/logstash/pipeline/52-winevent-system-output.conf index eafe1e4..9d3da93 100644 --- a/logstash/pipeline/52-winevent-system-output.conf +++ b/logstash/pipeline/52-winevent-system-output.conf @@ -1,8 +1,13 @@ +# HELK output conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + output { if [log_name] == "System"{ elasticsearch { hosts => ["elasticsearch:9200", "127.0.0.1:9200"] - sniffing => true manage_template => false index => "winevent-system-%{+YYYY.MM.dd}" } diff --git a/logstash/pipeline/53-winevent-application-output.conf b/logstash/pipeline/53-winevent-application-output.conf index 0752d1a..43b9608 100644 --- a/logstash/pipeline/53-winevent-application-output.conf +++ b/logstash/pipeline/53-winevent-application-output.conf @@ -1,8 +1,13 @@ +# HELK output conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + output { if [log_name] == "Application"{ elasticsearch { hosts => ["elasticsearch:9200", "127.0.0.1:9200"] - sniffing => true manage_template => false index => "winevent-application-%{+YYYY.MM.dd}" } diff --git a/logstash/pipeline/54-powershell-output.conf b/logstash/pipeline/54-powershell-output.conf index a6c6e3c..b9334bc 100644 --- a/logstash/pipeline/54-powershell-output.conf +++ b/logstash/pipeline/54-powershell-output.conf @@ -1,8 +1,13 @@ +# HELK output conf file +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + output { if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{ elasticsearch { hosts => ["elasticsearch:9200", "127.0.0.1:9200"] - sniffing => true manage_template => false index => "powershell-%{+YYYY.MM.dd}" } diff --git a/scripts/helk_docker_install.sh b/scripts/helk_docker_install.sh index 915ba36..6130c3b 100755 --- a/scripts/helk_docker_install.sh +++ b/scripts/helk_docker_install.sh @@ -1,10 +1,12 @@ #!/bin/bash -# Docker & Docker-compose Installation Script -# Author: Roberto Rodriguez @Cyb3rWard0g +# HELK script: helk_docker_install.sh +# HELK script description: Installs Docker & Docker-Compose on your HELK server. +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause -# Description: This script installs Docker & Docker-Compose on your HELK server. -# ELK version: 5x # References: # https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04 # https://www.digitalocean.com/community/tutorials/how-to-install-docker-compose-on-ubuntu-16-04 diff --git a/scripts/update_geoipdb.sh b/scripts/helk_geoipdb_update.sh similarity index 67% rename from scripts/update_geoipdb.sh rename to scripts/helk_geoipdb_update.sh index 7ef9fc2..a44ebd7 100755 --- a/scripts/update_geoipdb.sh +++ b/scripts/helk_geoipdb_update.sh @@ -1,9 +1,17 @@ #!/bin/bash + +# HELK script: helk_geoipdb_update.sh +# HELK script description: Update the MaxMind GeoIP databases +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + +# References: + # HELK Supporting script -# -# Original script from: SOF-ELK Supporting script (C)2016 Lewes Technology Consulting, LLC -# -# This script is used to update the MaxMind GeoIP databases +# SOF-ELK Supporting script (C)2016 Lewes Technology Consulting, LLC +# https://github.com/philhagen/sof-elk/blob/develop/supporting-scripts/geoip_update.sh GEOIP_LIBDIR=/etc/logstash/geoip GEOIP_CITYSOURCEURL=http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz diff --git a/scripts/helk_install.sh b/scripts/helk_install.sh index 4cf2dbd..9d04dd0 100755 --- a/scripts/helk_install.sh +++ b/scripts/helk_install.sh @@ -1,12 +1,14 @@ #!/bin/bash -# HELK Installation Script (Elasticsearch, Logstash, Kibana & Nginx) -# HELK build version: 0.9 (BETA Script) -# Author: Roberto Rodriguez @Cyb3rWard0g +# HELK script: helk_install.sh +# HELK script description: Install all the needed components of the HELK +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause -# Description: This script installs every single component of the ELK Stack plus Nginx -# ELK version: 5x -# Blog: https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html +# References: +# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html LOGFILE="/var/log/helk-install.log" @@ -47,7 +49,7 @@ ERROR=$? fi echo "[HELK INFO] Adding elastic packages source list definitions to your sources list.." -echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list >> $LOGFILE 2>&1 +echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list >> $LOGFILE 2>&1 ERROR=$? if [ $ERROR -ne 0 ]; then echoerror "Could not add elastic packages source list definitions to your source list (Error Code: $ERROR)." @@ -129,6 +131,9 @@ ERROR=$? echoerror "Could not start kibana and set kibana to start automatically when the system boots (Error Code: $ERROR)." fi +echo "[HELK INFO] Creating Index Patterns in Kibana" + + # *********** Installing Nginx *************** echo "[HELK INFO] Installing Nginx.." apt-get -y install nginx >> $LOGFILE 2>&1 @@ -219,9 +224,9 @@ ERROR=$? fi # *********** Creating Cron Job to run OTX script every monday at 8AM and capture last 30 days of Intel ************* echo "[HELK INFO] Creating a cronjob for OTX intel script" -mkdir /opt/helk/scripts -cp -v otx_helk.py /opt/helk/scripts/ -cronjob="0 8 * * 1 python /opt/helk/scripts/otx_helk.py" +mkdir /opt/helk/scripts >> $LOGFILE 2>&1 +cp -v helk_otx.py /opt/helk/scripts/ >> $LOGFILE 2>&1 +cronjob="0 8 * * 1 python /opt/helk/scripts/helk_otx.py" echo "$cronjob" | crontab - >> $LOGFILE 2>&1 ERROR=$? if [ $ERROR -ne 0 ]; then @@ -235,6 +240,14 @@ ERROR=$? if [ $ERROR -ne 0 ]; then echoerror "Could not install logstash (Error Code: $ERROR)." fi + +echo "[HELK INFO] Creating templates directory and copying custom templates over.." +mkdir /opt/helk/templates >> $LOGFILE 2>&1 +cp -v ../logstash/output_templates/* /opt/helk/templates/ >> $LOGFILE 2>&1 +ERROR=$? + if [ $ERROR -ne 0 ]; then + echoerror "Could not create templates directory and copy custom templates over (Error Code: $ERROR)." + fi echo "[HELK INFO] Copying logstash's .conf files.." cp -av ../logstash/pipeline/* /etc/logstash/conf.d/ >> $LOGFILE 2>&1 @@ -254,6 +267,14 @@ ERROR=$? echoerror "Could not start logstash and set it to start automatically when the system boots (Error Code: $ERROR)" fi +# *********** Create Kibana Index Patterns *************** +echo "[HELK INFO] Creating Kibana index patterns automatically.." +sh ./helk_kibana_index_pattern_creation.sh >> $LOGFILE 2>&1 +ERROR=$? + if [ $ERROR -ne 0 ]; then + echoerror "Could not create kibana index patterns (Error Code: $ERROR)." + fi + echo "**********************************************************************************************************" echo "[HELK INFO] Your HELK has been installed" echo "[HELK INFO] Browse to your host IP address from a different computer and enter the following credentials:" diff --git a/scripts/helk_kibana_index_pattern_creation.sh b/scripts/helk_kibana_index_pattern_creation.sh new file mode 100755 index 0000000..9e0fdc0 --- /dev/null +++ b/scripts/helk_kibana_index_pattern_creation.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# HELK script: helk_kibana_index_pattern_creation.sh +# HELK script description: Creates Kibana index patterns automatically. +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + +# References: +# https://github.com/elastic/kibana/issues/3709 (https://github.com/hobti01) +# https://explainshell.com/explain?cmd=set+-euxo%20pipefail + +set -euo pipefail +url="http://localhost:5601" +declare -A index_patterns=("sysmon-*" + "winevent-security-*" + "winevent-system-*" + "winevent-application-*" + "powershell-*" + ) +time_field="@timestamp" +# Create index pattern +# curl -f to fail on error +# For loop to create every single intex pattern +for index in ${!index_patterns[@]}; do + curl -f -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + "$url/api/saved_objects/index-pattern/${index}" \ + -d"{\"attributes\":{\"title\":\"${index}\",\"timeFieldName\":\"$time_field\"}}" +done +# Make Sysmon the default index +curl -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + "$url/api/kibana/settings/defaultIndex" \ + -d"{\"value\":\"sysmon-*\"}" \ No newline at end of file diff --git a/scripts/otx_helk.py b/scripts/helk_otx.py old mode 100644 new mode 100755 similarity index 91% rename from scripts/otx_helk.py rename to scripts/helk_otx.py index 696f381..026c984 --- a/scripts/otx_helk.py +++ b/scripts/helk_otx.py @@ -1,4 +1,12 @@ #!/usr/bin/env python + +# HELK script: helk_otx.py +# HELK script description: Pulling intelligence from OTX (AlienVault) +# HELK build version: 0.9 (BETA) +# HELK ELK version: 6.x +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: BSD 3-Clause + from OTXv2 import OTXv2 from pandas.io.json import json_normalize