HELK 6.2.3 - 032218

Docker-Compose file
+ Updated Image versions
++ helk-elk:6.2.3
++ helk-kafka:1.0.1
++ helk-analytics:0.0.2

HELK-ANALYTICS
+ Upgraded spark to version 2.3.0
++ Check release notes: https://spark.apache.org/releases/spark-release-2-3-0.html
+ Upgraded Jupyter Lab to 0.31.12
+ Downgraded Tornado to version 4.* This is due to an error in dependencies happening in version 5.0 with python 3.
+ Upgraded ES-Hadoop package to version 6.2.3
++ Check release notes:
https://www.elastic.co/guide/en/elasticsearch/hadoop/6.2/eshadoop-6.2.3.html

HELK-ELK
+ Upgraded elastic components to 6.2.3
++ Check elasticsearch release notes:
https://www.elastic.co/guide/en/elasticsearch/reference/6.2/release-notes-6.2.3.html
++ No changes for Kibana
++ Check Logstash release notes:
https://www.elastic.co/guide/en/logstash/6.2/logstash-6-2-3.html
+ Logstash kafka input now adds metadata from kafka. Topic name, etc.
+ Fingerprint plugin in logstash config 09-all-filter.con is applied to only events with the message field.
+ logstash config 11-winevent-sysmon-filter.conf
++ removed field "user". This was causing issues when parsing events with Spark.

HELK-KAFKA
+ Upgraded Kafka to version 2.11-1.0.1
++ Check kafka release notes:
https://www.apache.org/dist/kafka/1.0.1/RELEASE_NOTES.html
+ Removed sleep time for kafka init file
+ updated kafka entrypoint updating version values

HELK helk_install main script
+ Fixed docker & docker-compose installation steps. This fixes issue https://github.com/Cyb3rWard0g/HELK/issues/33

HELK Winlogbeat install script
+ Updated beat version to 6.2.3
keyword-vs-text-changes
Roberto Rodriguez 2018-03-22 03:32:21 -04:00
parent 669758782d
commit c43eaa08e3
15 changed files with 102 additions and 62 deletions

View File

@ -1,9 +1,9 @@
version: '3.2'
version: '3'
services:
helk-elk:
build: ./helk-elk
image: cyb3rward0g/helk-elk:6.2.3
container_name: helk-elk
volumes:
- esdata:/var/lib/elasticsearch
@ -25,7 +25,7 @@ services:
aliases:
- helk_elk.hunt.local
helk-kafka:
build: ./helk-kafka
image: cyb3rward0g/helk-kafka:1.0.1
container_name: helk-kafka
env_file: ./helk.env
ports:
@ -42,7 +42,7 @@ services:
aliases:
- helk_kafka.hunt.local
helk-analytics:
build: ./helk-analytics/
image: cyb3rward0g/helk-analytics:0.0.2
container_name: helk-analytics
ports:
- "8880:8880"

View File

@ -37,9 +37,9 @@ RUN pip3 install --upgrade pip
# *********** Installing HELK python packages ***************
RUN pip3 install \
pandas==0.22.0 \
tornado==4.* \
jupyter \
jupyterhub==0.8.1 \
jupyterlab==0.31.8 \
jupyterlab==0.31.12 \
https://dist.apache.org/repos/dist/dev/incubator/toree/0.2.0/snapshots/dev1/toree-pip/toree-0.2.0.dev1.tar.gz
RUN pip3 install scipy==1.0.0 \
@ -55,7 +55,7 @@ RUN pip3 install scipy==1.0.0 \
bokeh==0.12.14
# *********** Creating the right directories ***************
RUN bash -c 'mkdir -pv /opt/helk/{scripts,training,es-hadoop,spark,packages}'
RUN bash -c 'mkdir -pv /opt/helk/{scripts,training,es-hadoop,spark}'
# *********** Adding HELK scripts and files to Container ***************
ADD scripts/analytics-entrypoint.sh /opt/helk/scripts/
@ -63,7 +63,8 @@ RUN chmod +x /opt/helk/scripts/analytics-entrypoint.sh
ADD training/ /opt/helk/training/
# *********** Install ES-Hadoop ***************
RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-6.2.2.zip -P /opt/helk/es-hadoop/ \
ENV ESHADOOP_VERSION=6.2.3
RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-${ESHADOOP_VERSION}.zip -P /opt/helk/es-hadoop/ \
&& unzip /opt/helk/es-hadoop/*.zip -d /opt/helk/es-hadoop/ \
&& rm /opt/helk/es-hadoop/*.zip

View File

@ -31,6 +31,6 @@
# https://graphframes.github.io/quick-start.html
# https://spark-packages.org/package/graphframes/graphframes
spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.2.2/dist/elasticsearch-hadoop-6.2.2.jar
spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.2.3/dist/elasticsearch-hadoop-6.2.3.jar
spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3
#spark.python.profile true

View File

@ -1,6 +1,6 @@
# HELK script: HELK ELK Dockerfile
# HELK build version: 0.9 (ALPHA)
# HELK ELK version: 6.2.2
# HELK ELK version: 6.2.3
# Author: Roberto Rodriguez (@Cyb3rWard0g)
# License: BSD 3-Clause
@ -53,7 +53,7 @@ RUN chmod +x /opt/helk/scripts/elk-kibana-setup.sh
RUN chmod +x /opt/helk/scripts/elk-entrypoint.sh
# *********** ELK Version ***************
ENV ELK_VERSION=6.2.2
ENV ELK_VERSION=6.2.3
# *********** Installing Elasticsearch ***************
ENV ES_HELK_HOME=/opt/helk/elasticsearch

View File

@ -1,5 +1,5 @@
{
"version": "6.2.2",
"version": "6.2.3",
"objects": [
{
"id": "e351c080-1dd7-11e8-8f1b-1b86647d4817",
@ -236,7 +236,7 @@
"title": "Global_Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":3,\"y\":2,\"w\":4,\"h\":3,\"i\":\"1\"},\"id\":\"e351c080-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"97478120-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":3,\"y\":0,\"w\":3,\"h\":2,\"i\":\"4\"},\"id\":\"a5fe7110-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":2,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"bf617710-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"7\"},\"id\":\"24cc4b70-1dd8-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":3,\"y\":5,\"w\":3,\"h\":3,\"i\":\"8\"},\"id\":\"07d74510-1dd8-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"9\"},\"id\":\"32f92e60-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"10\"},\"id\":\"45159070-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"11\",\"gridData\":{\"x\":7,\"y\":2,\"w\":3,\"h\":3,\"i\":\"11\"},\"id\":\"9b6fe330-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"12\",\"gridData\":{\"x\":10,\"y\":2,\"w\":2,\"h\":3,\"i\":\"12\"},\"id\":\"ccdf5fe0-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"13\",\"gridData\":{\"x\":6,\"y\":5,\"w\":6,\"h\":3,\"i\":\"13\"},\"id\":\"cb8b5280-1de2-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"14\",\"gridData\":{\"x\":0,\"y\":8,\"w\":12,\"h\":7,\"i\":\"14\"},\"version\":\"6.2.2\",\"type\":\"search\",\"id\":\"0e899740-1de3-11e8-8f1b-1b86647d4817\"}]",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":3,\"y\":2,\"w\":4,\"h\":3,\"i\":\"1\"},\"id\":\"e351c080-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"97478120-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":3,\"y\":0,\"w\":3,\"h\":2,\"i\":\"4\"},\"id\":\"a5fe7110-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":2,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"bf617710-1dd7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"7\"},\"id\":\"24cc4b70-1dd8-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":3,\"y\":5,\"w\":3,\"h\":3,\"i\":\"8\"},\"id\":\"07d74510-1dd8-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"9\"},\"id\":\"32f92e60-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"10\"},\"id\":\"45159070-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"11\",\"gridData\":{\"x\":7,\"y\":2,\"w\":3,\"h\":3,\"i\":\"11\"},\"id\":\"9b6fe330-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"12\",\"gridData\":{\"x\":10,\"y\":2,\"w\":2,\"h\":3,\"i\":\"12\"},\"id\":\"ccdf5fe0-1dd9-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"13\",\"gridData\":{\"x\":6,\"y\":5,\"w\":6,\"h\":3,\"i\":\"13\"},\"id\":\"cb8b5280-1de2-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"14\",\"gridData\":{\"x\":0,\"y\":8,\"w\":12,\"h\":7,\"i\":\"14\"},\"version\":\"6.2.3\",\"type\":\"search\",\"id\":\"0e899740-1de3-11e8-8f1b-1b86647d4817\"}]",
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
"version": 1,
"timeRestore": true,

View File

@ -1,5 +1,5 @@
{
"version": "6.2.2",
"version": "6.2.3",
"objects": [
{
"id": "b2b6b460-1de3-11e8-8f1b-1b86647d4817",
@ -288,7 +288,7 @@
"title": "Sysmon_Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":4,\"y\":2,\"w\":5,\"h\":3,\"i\":\"1\"},\"id\":\"b2b6b460-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"40aab0b0-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":3,\"y\":0,\"w\":3,\"h\":2,\"i\":\"3\"},\"id\":\"55e73e80-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":7,\"y\":5,\"w\":5,\"h\":3,\"i\":\"4\"},\"id\":\"1f8837d0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"5\"},\"id\":\"68484ab0-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":7,\"y\":8,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"0c438260-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":3,\"y\":5,\"w\":4,\"h\":3,\"i\":\"7\"},\"id\":\"d36e8f20-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":10,\"y\":8,\"w\":2,\"h\":3,\"i\":\"8\"},\"id\":\"2ff90cc0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":9,\"y\":2,\"w\":3,\"h\":3,\"i\":\"9\"},\"id\":\"601666f0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"10\"},\"id\":\"f000dc10-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"11\",\"gridData\":{\"x\":0,\"y\":8,\"w\":7,\"h\":3,\"i\":\"11\"},\"id\":\"4a347160-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"12\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"12\"},\"id\":\"7c191380-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"13\",\"gridData\":{\"x\":0,\"y\":2,\"w\":4,\"h\":3,\"i\":\"13\"},\"id\":\"cb0bfe70-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"14\",\"gridData\":{\"x\":0,\"y\":14,\"w\":12,\"h\":6,\"i\":\"14\"},\"id\":\"2754df30-1de5-11e8-8f1b-1b86647d4817\",\"type\":\"search\",\"version\":\"6.2.2\"},{\"panelIndex\":\"15\",\"gridData\":{\"x\":0,\"y\":11,\"w\":6,\"h\":3,\"i\":\"15\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"c23c05f0-1de5-11e8-8f1b-1b86647d4817\"},{\"panelIndex\":\"16\",\"gridData\":{\"x\":6,\"y\":11,\"w\":6,\"h\":3,\"i\":\"16\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"fc7c21f0-1de5-11e8-8f1b-1b86647d4817\"}]",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":4,\"y\":2,\"w\":5,\"h\":3,\"i\":\"1\"},\"id\":\"b2b6b460-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"40aab0b0-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":3,\"y\":0,\"w\":3,\"h\":2,\"i\":\"3\"},\"id\":\"55e73e80-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":7,\"y\":5,\"w\":5,\"h\":3,\"i\":\"4\"},\"id\":\"1f8837d0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"5\"},\"id\":\"68484ab0-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":7,\"y\":8,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"0c438260-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":3,\"y\":5,\"w\":4,\"h\":3,\"i\":\"7\"},\"id\":\"d36e8f20-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":10,\"y\":8,\"w\":2,\"h\":3,\"i\":\"8\"},\"id\":\"2ff90cc0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":9,\"y\":2,\"w\":3,\"h\":3,\"i\":\"9\"},\"id\":\"601666f0-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"10\"},\"id\":\"f000dc10-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"11\",\"gridData\":{\"x\":0,\"y\":8,\"w\":7,\"h\":3,\"i\":\"11\"},\"id\":\"4a347160-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"12\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"12\"},\"id\":\"7c191380-1de3-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"13\",\"gridData\":{\"x\":0,\"y\":2,\"w\":4,\"h\":3,\"i\":\"13\"},\"id\":\"cb0bfe70-1de4-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"14\",\"gridData\":{\"x\":0,\"y\":14,\"w\":12,\"h\":6,\"i\":\"14\"},\"id\":\"2754df30-1de5-11e8-8f1b-1b86647d4817\",\"type\":\"search\",\"version\":\"6.2.3\"},{\"panelIndex\":\"15\",\"gridData\":{\"x\":0,\"y\":11,\"w\":6,\"h\":3,\"i\":\"15\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"c23c05f0-1de5-11e8-8f1b-1b86647d4817\"},{\"panelIndex\":\"16\",\"gridData\":{\"x\":6,\"y\":11,\"w\":6,\"h\":3,\"i\":\"16\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"fc7c21f0-1de5-11e8-8f1b-1b86647d4817\"}]",
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
"version": 1,
"timeRestore": true,

View File

@ -1,5 +1,5 @@
{
"version": "6.2.2",
"version": "6.2.3",
"objects": [
{
"id": "88ba6280-1de6-11e8-8f1b-1b86647d4817",
@ -59,7 +59,7 @@
"version": 1,
"attributes": {
"title": "Sysmon_Network_Map",
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"geoip.location\",\"isFilteredByCollar\":true,\"precision\":2,\"useGeocentroid\":true},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"params\":{\"addTooltip\":true,\"heatClusterSize\":2,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[0,0],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"baseLayersAreLoaded\":{},\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"selectedTmsLayer\":{\"attribution\":\"<p>&#169; <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p>&#10;\",\"id\":\"road_map\",\"maxZoom\":10,\"minZoom\":0,\"subdomains\":[],\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\"},\"tmsLayers\":[{\"attribution\":\"<p>&#169; <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p>&#10;\",\"id\":\"road_map\",\"maxZoom\":10,\"minZoom\":0,\"subdomains\":[],\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\"}]}},\"title\":\"Sysmon_Network_Map\",\"type\":\"tile_map\"}",
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"geoip.location\",\"isFilteredByCollar\":true,\"precision\":2,\"useGeocentroid\":true},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"params\":{\"addTooltip\":true,\"heatClusterSize\":2,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[0,0],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"baseLayersAreLoaded\":{},\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"selectedTmsLayer\":{\"attribution\":\"<p>&#169; <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p>&#10;\",\"id\":\"road_map\",\"maxZoom\":10,\"minZoom\":0,\"subdomains\":[],\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.3\"},\"tmsLayers\":[{\"attribution\":\"<p>&#169; <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p>&#10;\",\"id\":\"road_map\",\"maxZoom\":10,\"minZoom\":0,\"subdomains\":[],\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.3\"}]}},\"title\":\"Sysmon_Network_Map\",\"type\":\"tile_map\"}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "754acc80-1de6-11e8-8f1b-1b86647d4817",
@ -201,7 +201,7 @@
"title": "Sysmon_Network_Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"1\"},\"id\":\"88ba6280-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":9,\"y\":5,\"w\":3,\"h\":3,\"i\":\"2\"},\"id\":\"fea5c340-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"3\"},\"id\":\"9d5cac20-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":3,\"y\":0,\"w\":6,\"h\":5,\"i\":\"4\"},\"id\":\"d00f7e40-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":3,\"y\":5,\"w\":3,\"h\":3,\"i\":\"5\"},\"id\":\"e71b9bf0-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":2,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"5895e6f0-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":9,\"y\":2,\"w\":3,\"h\":3,\"i\":\"7\"},\"id\":\"70cca1f0-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":6,\"y\":5,\"w\":3,\"h\":3,\"i\":\"8\"},\"id\":\"8d4f5e80-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"9\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"bd839c10-1de7-11e8-8f1b-1b86647d4817\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":0,\"y\":8,\"w\":12,\"h\":7,\"i\":\"10\"},\"version\":\"6.2.2\",\"type\":\"search\",\"id\":\"754acc80-1de6-11e8-8f1b-1b86647d4817\"}]",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":3,\"h\":2,\"i\":\"1\"},\"id\":\"88ba6280-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":9,\"y\":5,\"w\":3,\"h\":3,\"i\":\"2\"},\"id\":\"fea5c340-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"3\"},\"id\":\"9d5cac20-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":3,\"y\":0,\"w\":6,\"h\":5,\"i\":\"4\"},\"id\":\"d00f7e40-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":3,\"y\":5,\"w\":3,\"h\":3,\"i\":\"5\"},\"id\":\"e71b9bf0-1de6-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":2,\"w\":3,\"h\":3,\"i\":\"6\"},\"id\":\"5895e6f0-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":9,\"y\":2,\"w\":3,\"h\":3,\"i\":\"7\"},\"id\":\"70cca1f0-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":6,\"y\":5,\"w\":3,\"h\":3,\"i\":\"8\"},\"id\":\"8d4f5e80-1de7-11e8-8f1b-1b86647d4817\",\"type\":\"visualization\",\"version\":\"6.2.3\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":0,\"y\":5,\"w\":3,\"h\":3,\"i\":\"9\"},\"version\":\"6.2.3\",\"type\":\"visualization\",\"id\":\"bd839c10-1de7-11e8-8f1b-1b86647d4817\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":0,\"y\":8,\"w\":12,\"h\":7,\"i\":\"10\"},\"version\":\"6.2.3\",\"type\":\"search\",\"id\":\"754acc80-1de6-11e8-8f1b-1b86647d4817\"}]",
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
"version": 1,
"timeRestore": true,

View File

@ -8,6 +8,7 @@ input {
{
bootstrap_servers => "172.18.0.3:9092,172.18.0.3:9093,172.18.0.3:9094"
topics => ["winlogbeat"]
decorate_events => true
codec => "json"
auto_offset_reset => "earliest"
############################# HELK Optimizing Latency #############################

View File

@ -4,10 +4,12 @@
# License: BSD 3-Clause
filter {
fingerprint {
source => "message"
target => "[@metadata][log_hash]"
method => "MURMUR3"
key => "Logstash"
if [message] {
fingerprint {
source => "message"
target => "[@metadata][log_hash]"
method => "MURMUR3"
key => "Logstash"
}
}
}

View File

@ -243,7 +243,7 @@ filter {
remove_field => "[event_data]"
remove_field => "message"
remove_field => "[event_data][Hashes]"
remove_field => "[event_data][User]"
remove_field => "[user]"
}
}
}

View File

@ -7,6 +7,8 @@ FROM phusion/baseimage
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
LABEL description="Dockerfile base for the HELK Kafka."
USER root
ENV DEBIAN_FRONTEND noninteractive
# *********** Installing Prerequisites ***************
@ -30,14 +32,20 @@ RUN bash -c 'mkdir -pv /opt/helk/{scripts,kafka}'
# *********** Install Kafka ***************
ENV KAFKA_LOGS_PATH=/var/log/kafka
RUN wget -qO- http://apache.mirrors.lucidnetworks.net/kafka/1.0.0/kafka_2.11-1.0.0.tgz | sudo tar xvz -C /opt/helk/kafka/ \
ENV KAFKA_HOME=/opt/helk/kafka/kafka_2.11-1.0.1
RUN wget -qO- http://mirrors.advancedhosters.com/apache/kafka/1.0.1/kafka_2.11-1.0.1.tgz | sudo tar xvz -C /opt/helk/kafka/ \
&& mkdir -v $KAFKA_LOGS_PATH \
&& mv /opt/helk/kafka/kafka_2.11-1.0.0/config/server.properties /opt/helk/kafka/kafka_2.11-1.0.0/config/backup_server.properties
ADD *.properties /opt/helk/kafka/kafka_2.11-1.0.0/config/
&& mv ${KAFKA_HOME}/config/server.properties ${KAFKA_HOME}/config/backup_server.properties
ADD *.properties ${KAFKA_HOME}/config/
ADD kafka-init /etc/init.d/kafka
ADD scripts/kafka-entrypoint.sh /opt/helk/scripts/
RUN chmod +x /opt/helk/scripts/kafka-entrypoint.sh
# *********** Download KSQL ******************************
ENV KSQL_VERSION=0.4
RUN wget -qO- https://github.com/confluentinc/ksql/releases/download/v${KSQL_VERSION}/ksql-${KSQL_VERSION}.tgz | sudo tar xvz -C /opt/helk/
EXPOSE 2181 9092 9093 9094
WORKDIR "/opt/helk/scripts/"
ENTRYPOINT ["./kafka-entrypoint.sh"]

View File

@ -32,7 +32,7 @@ if [ -r /etc/default/rcS ]; then
. /etc/default/rcS
fi
KAFKA_HOME=/opt/helk/kafka/kafka_2.11-1.0.0
KAFKA_HOME=/opt/helk/kafka/kafka_2.11-1.0.1
KAFKA_USER=root
KAFKA_GROUP=root
KAFKA_NICE=18
@ -88,7 +88,6 @@ start() {
# and a process possibly asking for status.
echo $! > $KAFKA_PIDFILE_ZOOKEEPER
echo "Kafka Zookeeper started."
sleep 5
declare -a pids=($KAFKA_PIDFILE $KAFKA_PIDFILE_1 $KAFKA_PIDFILE_2)
for p in ${pids[@]}; do

View File

@ -6,6 +6,8 @@
# Author: Roberto Rodriguez (@Cyb3rWard0g)
# License: BSD 3-Clause
KAFKA_VERSION=2.11-1.0.1
# Start graceful termination of HELK services that might be running before running the entrypoint script.
_term() {
echo "Terminating HELK-Kafka Service"
@ -22,14 +24,14 @@ rm -f /var/run/kafka_zookeeper.pid \
# *********** Start Kafka **************
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting current host IP to brokers server.properties files.."
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9092/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9092/g" /opt/helk/kafka/kafka_2.11-1.0.0/config/server.properties
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9093/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9093/g" /opt/helk/kafka/kafka_2.11-1.0.0/config/server-1.properties
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9094/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9094/g" /opt/helk/kafka/kafka_2.11-1.0.0/config/server-2.properties
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9092/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9092/g" /opt/helk/kafka/kafka_${KAFKA_VERSION}/config/server.properties
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9093/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9093/g" /opt/helk/kafka/kafka_${KAFKA_VERSION}/config/server-1.properties
sed -i "s/advertised\.listeners\=PLAINTEXT:\/\/HELKIP\:9094/advertised\.listeners\=PLAINTEXT\:\/\/${ADVERTISED_LISTENER}\:9094/g" /opt/helk/kafka/kafka_${KAFKA_VERSION}/config/server-2.properties
echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Kafka.."
service kafka start
sleep 30
echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka Winlogbeat Topic.."
/opt/helk/kafka/kafka_2.11-1.0.0/bin/kafka-topics.sh --create --zookeeper $ADVERTISED_LISTENER:2181 --replication-factor 3 --partitions 1 --topic winlogbeat
echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka winlogbeat Topic.."
/opt/helk/kafka/kafka_${KAFKA_VERSION}/bin/kafka-topics.sh --create --zookeeper $ADVERTISED_LISTENER:2181 --replication-factor 3 --partitions 1 --topic winlogbeat
echo "[HELK-DOCKER-INSTALLATION-INFO] Pushing Spark Logs to console.."
tail -f /var/log/kafka/helk-*.log

View File

@ -3,7 +3,7 @@
# HELK script: helk_install.sh
# HELK script description: Start
# HELK build version: 0.9 (Alpha)
# HELK ELK version: 6.2.2
# HELK ELK version: 6.2.3
# Author: Roberto Rodriguez (@Cyb3rWard0g)
# License: BSD 3-Clause
@ -30,13 +30,39 @@ get_token(){
jupyter_token="$(docker exec -ti helk-analytics jupyter notebook list | grep -oP '(?<=token=).*(?= ::)' | awk '{$1=$1};1')" >> $LOGFILE 2>&1
}
# ********** Install Curl ********************
install_curl(){
echo "[HELK-INSTALLATION-INFO] Checking if curl is installed first"
if [ -x "$(command -v curl)" ]; then
echo "[HELK-INSTALLATION-INFO] curl is already installed"
else
echo "[HELK-INSTALLATION-INFO] curl is not installed"
echo "[HELK-INSTALLATION-INFO] Installing curl before installing docker.."
apt-get install -y curl >> $LOGFILE 2>&1
ERROR=$?
if [ $ERROR -ne 0 ]; then
echoerror "Could not install curl (Error Code: $ERROR)."
exit 1
fi
fi
}
# *********** Building and Running HELK Images ***************
build_run(){
echo "[HELK-INSTALLATION-INFO] Installing HELK via docker-compose"
echo "[HELK-INSTALLATION-INFO] Building HELK via docker-compose"
echo "ADVERTISED_LISTENER=$host_ip" >> helk.env
docker-compose up -d >> $LOGFILE 2>&1
docker-compose build >> $LOGFILE 2>&1
ERROR=$?
if [ $ERROR -ne 0 ]; then
echoerror "Could not build HELK via docker-compose (Error Code: $ERROR)."
echo "get more details in /var/log/helk-install.log locally"
exit 1
fi
echo "[HELK-INSTALLATION-INFO] Running HELK via docker-compose"
docker-compose up -d >> $LOGFILE 2>&1
ERROR=$?
if [ $ERROR -ne 0 ]; then
echoerror "Could not run HELK via docker-compose (Error Code: $ERROR)."
exit 1
fi
}
@ -49,7 +75,7 @@ show_banner() {
echo "** **"
echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **"
echo "** HELK build version: 0.9 (Alpha) **"
echo "** HELK ELK version: 6.2.2 **"
echo "** HELK ELK version: 6.2.3 **"
echo "** License: BSD 3-Clause **"
echo "**********************************************"
echo " "
@ -116,23 +142,13 @@ prepare_helk(){
# *********** Check if docker is installed ***************
if [ -x "$(command -v docker)" ]; then
echo "[HELK-INSTALLATION-INFO] Docker already installed"
echo "[HELK-INSTALLATION-INFO] Dockerizing HELK.."
else
echo "[HELK-INSTALLATION-INFO] Docker is not installed"
echo "[HELK-INSTALLATION-INFO] Checking if curl is installed first"
if [ -x "$(command -v curl)" ]; then
echo "[HELK-INSTALLATION-INFO] curl is already installed"
echo "[HELK-INSTALLATION-INFO] Ready to install Docker.."
else
echo "[HELK-INSTALLATION-INFO] curl is not installed"
echo "[HELK-INSTALLATION-INFO] Installing curl before installing docker.."
apt-get install -y curl >> $LOGFILE 2>&1
ERROR=$?
if [ $ERROR -ne 0 ]; then
echoerror "Could not install curl (Error Code: $ERROR)."
exit 1
fi
fi
# ****** Install Curl if it is not installed *********
install_curl
# ****** Installing via convenience script ***********
echo "[HELK-INSTALLATION-INFO] Installing docker via convenience script.."
curl -fsSL get.docker.com -o scripts/get-docker.sh >> $LOGFILE 2>&1
@ -143,6 +159,16 @@ prepare_helk(){
echoerror "Could not install docker via convenience script (Error Code: $ERROR)."
exit 1
fi
fi
# ********** Check if docker-compose is installed *******
if [ -x "$(command -v docker-compose)" ]; then
echo "[HELK-INSTALLATION-INFO] Docker-compose already installed"
else
echo "[HELK-INSTALLATION-INFO] Docker-compose is not installed"
# ****** Install Curl if it is not installed *********
install_curl
# ****** Installing docker-compose ***********
echo "[HELK-INSTALLATION-INFO] Installing docker-compose .."
curl -L https://github.com/docker/compose/releases/download/1.19.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose >> $LOGFILE 2>&1
@ -155,14 +181,14 @@ prepare_helk(){
fi
else
# *********** Check if docker is installed ***************
if [ -x "$(command -v docker)" ]; then
echo "[HELK-INSTALLATION-INFO] Docker already installed"
echo "[HELK-INSTALLATION-INFO] Dockerizing HELK.."
if [ -x "$(command -v docker)" ] && [ -x "$(command -v docker-compose)" ]; then
echo "[HELK-INSTALLATION-INFO] Docker & Docker-compose already installed"
else
echo "[HELK-INSTALLATION-INFO] Install docker for $systemKernel"
echo "[HELK-INSTALLATION-INFO] Install Docker & Docker-compose for $systemKernel"
exit 1
fi
fi
echo "[HELK-INSTALLATION-INFO] Dockerizing HELK.."
echo "[HELK-INSTALLATION-INFO] Checking local vm.max_map_count variable and setting it to 262144"
MAX_MAP_COUNT=262144
if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
@ -205,6 +231,7 @@ show_banner
prepare_helk
build_run
get_token
sleep 20
echo " "
echo " "

View File

@ -29,10 +29,10 @@ function start-winlogbeat
}
}
$winInstall_source = "https://artifacts.elastic.co/downloads/beats/winlogbeat/winlogbeat-6.0.0-windows-x86_64.zip"
$winInstall_dest = ($env:ProgramFiles + "\winlogbeat-6.0.0-windows-x86_64.zip")
$winInstall_source = "https://artifacts.elastic.co/downloads/beats/winlogbeat/winlogbeat-6.2.3-windows-x86_64.zip"
$winInstall_dest = ($env:ProgramFiles + "\winlogbeat-6.2.3-windows-x86_64.zip")
$winconfig_dest = ($env:ProgramFiles + "\winlogbeat\winlogbeat.yml")
$winInstall_old = $env:ProgramFiles + "\winlogbeat-6.0.0-windows-x86_64"
$winInstall_old = $env:ProgramFiles + "\winlogbeat-6.2.3-windows-x86_64"
$winInstall_new = $env:ProgramFiles + "\winlogbeat"
if (Get-WmiObject -class win32_service | Where-Object {$_.Name -like "winlogbeat"})