mirror of https://github.com/infosecn1nja/HELK.git
Merged conflicting changes
Dev Dua
commit
03a32bb2bf
|
|
@ -1,16 +1,23 @@
|
|||
version: '3'
|
||||
version: '3.5'
|
||||
|
||||
services:
|
||||
helk-elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0
|
||||
container_name: helk-elasticsearch
|
||||
secrets:
|
||||
- source: elasticsearch.yml
|
||||
target: /usr/share/elasticsearch/config/elasticsearch.yml
|
||||
volumes:
|
||||
- ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
- ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
|
||||
entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
|
||||
environment:
|
||||
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
|
||||
- cluster.name=helk-cluster
|
||||
- node.name=helk-1
|
||||
- bootstrap.memory_lock=true
|
||||
- discovery.zen.minimum_master_nodes=1
|
||||
- discovery.type=single-node
|
||||
- "ES_JAVA_OPTS=-Xms2g -Xmx2g"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
|
|
@ -18,13 +25,13 @@ services:
|
|||
restart: always
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_elasticsearch.hunt.local
|
||||
helk-logstash:
|
||||
image: docker.elastic.co/logstash/logstash:6.3.2
|
||||
image: docker.elastic.co/logstash/logstash:6.4.0
|
||||
container_name: helk-logstash
|
||||
secrets:
|
||||
- source: logstash.yml
|
||||
target: /usr/share/logstash/config/logstash.yml
|
||||
volumes:
|
||||
- ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
|
||||
- ./helk-logstash/pipeline:/usr/share/logstash/pipeline
|
||||
- ./helk-logstash/output_templates:/usr/share/logstash/output_templates
|
||||
- ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
|
||||
|
|
@ -32,18 +39,20 @@ services:
|
|||
environment:
|
||||
- "LS_JAVA_OPTS=-Xms1g -Xmx1g"
|
||||
entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
|
||||
ports:
|
||||
- "5044:5044"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-elasticsearch
|
||||
- helk-zookeeper
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_logstash.hunt.local
|
||||
helk-kibana:
|
||||
image: docker.elastic.co/kibana/kibana:6.3.2
|
||||
image: docker.elastic.co/kibana/kibana:6.4.0
|
||||
container_name: helk-kibana
|
||||
secrets:
|
||||
- source: kibana.yml
|
||||
target: /usr/share/kibana/config/kibana.yml
|
||||
volumes:
|
||||
- ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
|
||||
- ./helk-kibana/dashboards:/usr/share/kibana/dashboards
|
||||
- ./helk-kibana/scripts:/usr/share/kibana/scripts
|
||||
entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh
|
||||
|
|
@ -52,13 +61,13 @@ services:
|
|||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kibana.hunt.local
|
||||
helk-nginx:
|
||||
image: cyb3rward0g/helk-nginx:0.0.6
|
||||
image: cyb3rward0g/helk-nginx:0.0.7
|
||||
container_name: helk-nginx
|
||||
secrets:
|
||||
- source: htpasswd.users
|
||||
target: /etc/nginx/htpasswd.users
|
||||
volumes:
|
||||
- ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users
|
||||
- ./helk-nginx/default:/etc/nginx/sites-available/default
|
||||
- ./helk-nginx/scripts/:/opt/helk/scripts/
|
||||
entrypoint: /opt/helk/scripts/nginx-entrypoint.sh
|
||||
|
|
@ -68,22 +77,19 @@ services:
|
|||
restart: always
|
||||
depends_on:
|
||||
- helk-kibana
|
||||
- helk-jupyter
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_nginx.hunt.local
|
||||
helk-jupyter:
|
||||
image: cyb3rward0g/helk-jupyter:0.0.4
|
||||
image: cyb3rward0g/helk-jupyter:0.0.5
|
||||
container_name: helk-jupyter
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-nginx
|
||||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_jupyter.hunt.local
|
||||
helk-spark-master:
|
||||
image: cyb3rward0g/helk-spark-master:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-master:2.3.1-b
|
||||
container_name: helk-spark-master
|
||||
environment:
|
||||
- SPARK_MASTER_PORT=7077
|
||||
|
|
@ -95,44 +101,34 @@ services:
|
|||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_master.hunt.local
|
||||
helk-spark-worker:
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-b
|
||||
container_name: helk-spark-worker
|
||||
environment:
|
||||
- SPARK_MASTER=spark://helk-spark-master:7077
|
||||
- SPARK_WORKER_MEMORY=512m
|
||||
- SPARK_WORKER_WEBUI_PORT=8081
|
||||
- SPARK_WORKER_PORT=42950
|
||||
ports:
|
||||
- "8081:8081"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-spark-master
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_worker.hunt.local
|
||||
helk-spark-worker2:
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-b
|
||||
container_name: helk-spark-worker2
|
||||
environment:
|
||||
- SPARK_MASTER=spark://helk-spark-master:7077
|
||||
- SPARK_WORKER_MEMORY=512m
|
||||
- SPARK_WORKER_WEBUI_PORT=8082
|
||||
- SPARK_WORKER_PORT=42951
|
||||
ports:
|
||||
- "8082:8082"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-spark-master
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_worker2.hunt.local
|
||||
helk-zookeeper:
|
||||
image: cyb3rward0g/helk-zookeeper:1.1.1
|
||||
image: cyb3rward0g/helk-zookeeper:2.0.0-a
|
||||
container_name: helk-zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
|
|
@ -141,10 +137,8 @@ services:
|
|||
- helk-kibana
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_zookeeper.hunt.local
|
||||
helk-kafka-broker:
|
||||
image: cyb3rward0g/helk-kafka-broker:1.1.1
|
||||
image: cyb3rward0g/helk-kafka-broker:2.0.0-b
|
||||
container_name: helk-kafka-broker
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -156,14 +150,13 @@ services:
|
|||
- REPLICATION_FACTOR=2
|
||||
- ADVERTISED_LISTENER=HOSTIP
|
||||
- ZOOKEEPER_NAME=helk-zookeeper
|
||||
- KAFKA_CREATE_TOPICS=True
|
||||
ports:
|
||||
- "9092:9092"
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kafka_broker.hunt.local
|
||||
helk-kafka-broker2:
|
||||
image: cyb3rward0g/helk-kafka-broker:1.1.1
|
||||
image: cyb3rward0g/helk-kafka-broker:2.0.0-b
|
||||
container_name: helk-kafka-broker2
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -175,25 +168,26 @@ services:
|
|||
- REPLICATION_FACTOR=2
|
||||
- ADVERTISED_LISTENER=HOSTIP
|
||||
- ZOOKEEPER_NAME=helk-zookeeper
|
||||
- KAFKA_CREATE_TOPICS=True
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kafka_broker2.hunt.local
|
||||
helk-sigma:
|
||||
image: thomaspatzke/helk-sigma
|
||||
container_name: helk-sigma
|
||||
depends_on:
|
||||
- helk-kibana
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_sigma.hunt.local
|
||||
|
||||
networks:
|
||||
helk:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
esdata:
|
||||
driver: local
|
||||
driver: local
|
||||
|
||||
secrets:
|
||||
elasticsearch.yml:
|
||||
file: ./helk-elasticsearch/elasticsearch.yml
|
||||
logstash.yml:
|
||||
file: ./helk-logstash/logstash.yml
|
||||
kibana.yml:
|
||||
file: ./helk-kibana/kibana.yml
|
||||
htpasswd.users:
|
||||
file: ./helk-nginx/htpasswd.users
|
||||
|
|
@ -1,16 +1,24 @@
|
|||
version: '3'
|
||||
version: '3.5'
|
||||
|
||||
services:
|
||||
helk-elasticsearch:
|
||||
build: helk-elasticsearch/trial/
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0
|
||||
container_name: helk-elasticsearch
|
||||
secrets:
|
||||
- source: elasticsearch.yml
|
||||
target: /usr/share/elasticsearch/config/elasticsearch.yml
|
||||
volumes:
|
||||
- ./helk-elasticsearch/trial/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
- ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts
|
||||
entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh
|
||||
environment:
|
||||
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
|
||||
- cluster.name=helk-cluster
|
||||
- node.name=helk-1
|
||||
- bootstrap.memory_lock=true
|
||||
- discovery.zen.minimum_master_nodes=1
|
||||
- discovery.type=single-node
|
||||
- "ES_JAVA_OPTS=-Xms2g -Xmx2g"
|
||||
- ELASTIC_PASSWORD=elasticpassword
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
|
|
@ -18,13 +26,13 @@ services:
|
|||
restart: always
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_elasticsearch.hunt.local
|
||||
helk-logstash:
|
||||
image: docker.elastic.co/logstash/logstash:6.3.2
|
||||
image: docker.elastic.co/logstash/logstash:6.4.0
|
||||
container_name: helk-logstash
|
||||
secrets:
|
||||
- source: logstash.yml
|
||||
target: /usr/share/logstash/config/logstash.yml
|
||||
volumes:
|
||||
- ./helk-logstash/trial/logstash.yml:/usr/share/logstash/config/logstash.yml
|
||||
- ./helk-logstash/trial/pipeline:/usr/share/logstash/pipeline
|
||||
- ./helk-logstash/output_templates:/usr/share/logstash/output_templates
|
||||
- ./helk-logstash/enrichments/cti:/usr/share/logstash/cti
|
||||
|
|
@ -32,18 +40,20 @@ services:
|
|||
environment:
|
||||
- "LS_JAVA_OPTS=-Xms1g -Xmx1g"
|
||||
entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh
|
||||
ports:
|
||||
- "5044:5044"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-kibana
|
||||
- helk-zookeeper
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_logstash.hunt.local
|
||||
helk-kibana:
|
||||
image: docker.elastic.co/kibana/kibana:6.3.2
|
||||
image: docker.elastic.co/kibana/kibana:6.4.0
|
||||
container_name: helk-kibana
|
||||
secrets:
|
||||
- source: kibana.yml
|
||||
target: /usr/share/kibana/config/kibana.yml
|
||||
volumes:
|
||||
- ./helk-kibana/trial/kibana.yml:/usr/share/kibana/config/kibana.yml
|
||||
- ./helk-kibana/dashboards:/usr/share/kibana/dashboards
|
||||
- ./helk-kibana/trial/scripts:/usr/share/kibana/scripts
|
||||
entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh
|
||||
|
|
@ -52,10 +62,8 @@ services:
|
|||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kibana.hunt.local
|
||||
helk-nginx:
|
||||
image: cyb3rward0g/helk-nginx:0.0.6
|
||||
image: cyb3rward0g/helk-nginx:0.0.7
|
||||
container_name: helk-nginx
|
||||
volumes:
|
||||
- ./helk-nginx/trial/default:/etc/nginx/sites-available/default
|
||||
|
|
@ -67,22 +75,19 @@ services:
|
|||
restart: always
|
||||
depends_on:
|
||||
- helk-kibana
|
||||
- helk-jupyter
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_nginx.hunt.local
|
||||
helk-jupyter:
|
||||
image: cyb3rward0g/helk-jupyter:0.0.4
|
||||
image: cyb3rward0g/helk-jupyter:0.0.5
|
||||
container_name: helk-jupyter
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-nginx
|
||||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_jupyter.hunt.local
|
||||
helk-spark-master:
|
||||
image: cyb3rward0g/helk-spark-master:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-master:2.3.1-b
|
||||
container_name: helk-spark-master
|
||||
environment:
|
||||
- SPARK_MASTER_PORT=7077
|
||||
|
|
@ -94,56 +99,44 @@ services:
|
|||
- helk-elasticsearch
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_master.hunt.local
|
||||
helk-spark-worker:
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-b
|
||||
container_name: helk-spark-worker
|
||||
environment:
|
||||
- SPARK_MASTER=spark://helk-spark-master:7077
|
||||
- SPARK_WORKER_MEMORY=512m
|
||||
- SPARK_WORKER_WEBUI_PORT=8081
|
||||
- SPARK_WORKER_PORT=42950
|
||||
ports:
|
||||
- "8081:8081"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-spark-master
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_worker.hunt.local
|
||||
helk-spark-worker2:
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-a
|
||||
image: cyb3rward0g/helk-spark-worker:2.3.1-b
|
||||
container_name: helk-spark-worker2
|
||||
environment:
|
||||
- SPARK_MASTER=spark://helk-spark-master:7077
|
||||
- SPARK_WORKER_MEMORY=512m
|
||||
- SPARK_WORKER_WEBUI_PORT=8082
|
||||
- SPARK_WORKER_PORT=42951
|
||||
ports:
|
||||
- "8082:8082"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-spark-master
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_spark_worker2.hunt.local
|
||||
helk-zookeeper:
|
||||
image: cyb3rward0g/helk-zookeeper:1.1.1
|
||||
image: cyb3rward0g/helk-zookeeper:2.0.0-a
|
||||
container_name: helk-zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
restart: always
|
||||
depends_on:
|
||||
- helk-elasticsearch
|
||||
- helk-kibana
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_zookeeper.hunt.local
|
||||
helk-kafka-broker:
|
||||
image: cyb3rward0g/helk-kafka-broker:1.1.1
|
||||
image: cyb3rward0g/helk-kafka-broker:2.0.0-b
|
||||
container_name: helk-kafka-broker
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -155,14 +148,13 @@ services:
|
|||
- REPLICATION_FACTOR=2
|
||||
- ADVERTISED_LISTENER=HOSTIP
|
||||
- ZOOKEEPER_NAME=helk-zookeeper
|
||||
- KAFKA_CREATE_TOPICS=True
|
||||
ports:
|
||||
- "9092:9092"
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kafka_broker.hunt.local
|
||||
helk-kafka-broker2:
|
||||
image: cyb3rward0g/helk-kafka-broker:1.1.1
|
||||
image: cyb3rward0g/helk-kafka-broker:2.0.0-b
|
||||
container_name: helk-kafka-broker2
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -174,21 +166,12 @@ services:
|
|||
- REPLICATION_FACTOR=2
|
||||
- ADVERTISED_LISTENER=HOSTIP
|
||||
- ZOOKEEPER_NAME=helk-zookeeper
|
||||
- KAFKA_CREATE_TOPICS=True
|
||||
ports:
|
||||
- "9093:9093"
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_kafka_broker.hunt.local
|
||||
helk-sigma:
|
||||
image: thomaspatzke/helk-sigma
|
||||
container_name: helk-sigma
|
||||
depends_on:
|
||||
- helk-kibana
|
||||
networks:
|
||||
helk:
|
||||
aliases:
|
||||
- helk_sigma.hunt.local
|
||||
|
||||
networks:
|
||||
helk:
|
||||
driver: bridge
|
||||
|
|
@ -196,4 +179,12 @@ networks:
|
|||
volumes:
|
||||
esdata:
|
||||
driver: local
|
||||
|
||||
secrets:
|
||||
elasticsearch.yml:
|
||||
file: ./helk-elasticsearch/trial/elasticsearch.yml
|
||||
logstash.yml:
|
||||
file: ./helk-logstash/trial/logstash.yml
|
||||
kibana.yml:
|
||||
file: ./helk-kibana/trial/kibana.yml
|
||||
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM phusion/baseimage:0.10.1
|
||||
FROM phusion/baseimage:0.11
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile HELK Base Image.."
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
# HELK script: HELK Elasticsearch Dockerfile
|
||||
# HELK build Stage: Alpha
|
||||
# HELK ELK version: 6.3.2
|
||||
# HELK ELK version: 6.4.0
|
||||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
# References:
|
||||
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
|
||||
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:6.4.0
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Elasticsearch."
|
||||
|
|
@ -14,13 +14,13 @@
|
|||
#
|
||||
# Use a descriptive name for your cluster:
|
||||
#
|
||||
cluster.name: helk-elk
|
||||
#cluster.name: helk-elk
|
||||
#
|
||||
# ------------------------------------ Node ------------------------------------
|
||||
#
|
||||
# Use a descriptive name for the node:
|
||||
#
|
||||
node.name: helk-1
|
||||
#node.name: helk-1
|
||||
#
|
||||
# Add custom attributes to the node:
|
||||
#
|
||||
|
|
@ -40,7 +40,7 @@ node.name: helk-1
|
|||
#
|
||||
# Lock the memory on startup:
|
||||
#
|
||||
bootstrap.memory_lock: true
|
||||
#bootstrap.memory_lock: true
|
||||
#
|
||||
# Make sure that the heap size is set to about half the memory available
|
||||
# on the system and that the owner of the process is allowed to use this
|
||||
|
|
@ -74,8 +74,8 @@ network.host: 0.0.0.0
|
|||
# minimum_master_nodes need to be explicitly set when bound on a public IP
|
||||
# set to 1 to allow single node clusters
|
||||
# Details: https://github.com/elastic/elasticsearch/pull/17288
|
||||
discovery.zen.minimum_master_nodes: 1
|
||||
discovery.type: single-node
|
||||
#discovery.zen.minimum_master_nodes: 1
|
||||
#discovery.type: single-node
|
||||
#
|
||||
# For more information, consult the zen discovery module documentation.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
# HELK script: HELK Elasticsearch Dockerfile
|
||||
# HELK build Stage: Alpha
|
||||
# HELK ELK version: 6.3.2
|
||||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
# References:
|
||||
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
|
||||
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Elasticsearch."
|
||||
|
||||
RUN /usr/share/elasticsearch/bin/elasticsearch-keystore create
|
||||
RUN printf "elasticpassword" | /usr/share/elasticsearch/bin/elasticsearch-keystore add "bootstrap.password"
|
||||
|
|
@ -14,13 +14,13 @@
|
|||
#
|
||||
# Use a descriptive name for your cluster:
|
||||
#
|
||||
cluster.name: helk-elk
|
||||
#cluster.name: helk-elk
|
||||
#
|
||||
# ------------------------------------ Node ------------------------------------
|
||||
#
|
||||
# Use a descriptive name for the node:
|
||||
#
|
||||
node.name: helk-1
|
||||
#node.name: helk-1
|
||||
#
|
||||
# Add custom attributes to the node:
|
||||
#
|
||||
|
|
@ -40,7 +40,7 @@ node.name: helk-1
|
|||
#
|
||||
# Lock the memory on startup:
|
||||
#
|
||||
bootstrap.memory_lock: true
|
||||
#bootstrap.memory_lock: true
|
||||
#
|
||||
# Make sure that the heap size is set to about half the memory available
|
||||
# on the system and that the owner of the process is allowed to use this
|
||||
|
|
@ -74,8 +74,8 @@ network.host: 0.0.0.0
|
|||
# minimum_master_nodes need to be explicitly set when bound on a public IP
|
||||
# set to 1 to allow single node clusters
|
||||
# Details: https://github.com/elastic/elasticsearch/pull/17288
|
||||
discovery.zen.minimum_master_nodes: 1
|
||||
discovery.type: single-node
|
||||
#discovery.zen.minimum_master_nodes: 1
|
||||
#discovery.type: single-node
|
||||
#
|
||||
# For more information, consult the zen discovery module documentation.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -3,16 +3,16 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1-a
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for HELK Jupyter."
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
USER root
|
||||
|
||||
# *********** Installing Prerequisites ***************
|
||||
# -qq : No output except for errors
|
||||
RUN echo "[HELK-DOCKER-INSTALLATION-INFO] Updating Ubuntu base image.." \
|
||||
&& apt-get update -qq
|
||||
RUN echo "[HELK-DOCKER-INSTALLATION-INFO] Extracting templates from packages.." \
|
||||
&& apt-get install -qqy --no-install-recommends \
|
||||
curl python3-pip python3-dev python-tk unzip python3-setuptools \
|
||||
|
|
@ -30,14 +30,17 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
|
|||
RUN apt-get install -y --no-install-recommends nodejs
|
||||
|
||||
# *********** Installing HELK python packages ***************
|
||||
RUN pip3 install pandas \
|
||||
jupyter \
|
||||
jupyterlab==0.33.4 \
|
||||
jupyterhub==0.9.1
|
||||
RUN pip3 install \
|
||||
pandas==0.23.4 \
|
||||
altair==2.2.2 \
|
||||
vega_datasets \
|
||||
jupyter==1.0.0 \
|
||||
jupyterlab==0.34.1 \
|
||||
jupyterhub==0.9.2
|
||||
|
||||
# *********** Installing Jupyter Lab Extension - JupyterHub ***************
|
||||
RUN npm install -g configurable-http-proxy
|
||||
RUN jupyter labextension install @jupyterlab/hub-extension@0.10.0
|
||||
RUN jupyter labextension install @jupyterlab/hub-extension@0.11.0
|
||||
|
||||
# *********** Creating the Jupyter directories ***************
|
||||
RUN bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter,jupyterhub}'
|
||||
|
|
@ -60,6 +63,11 @@ RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsear
|
|||
&& unzip -j /opt/helk/es-hadoop/*.zip -d /opt/helk/es-hadoop/ \
|
||||
&& rm /opt/helk/es-hadoop/*.zip
|
||||
|
||||
# *********** Download Graphframes ***************
|
||||
#ENV GRAPHFRAMES_VERSION=0.6.0
|
||||
#RUN wget -qO- https://github.com/graphframes/graphframes/archive/release-${GRAPHFRAMES_VERSION}.tar.gz | sudo tar xvz -C /opt/helk/graphframes/ --strip-components=1 \
|
||||
# && mv /opt/helk/graphframes/python/graphframes /opt/helk/spark/python/pyspark/graphframes
|
||||
|
||||
EXPOSE 8000
|
||||
# *********** RUN HELK ***************
|
||||
WORKDIR ${JUPYTER_DIR}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval
|
|||
spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.3.2.jar
|
||||
# Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths.
|
||||
# The coordinates should be groupId:artifactId:version.
|
||||
spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3
|
||||
spark.jars.packages graphframes:graphframes:0.6.0-spark2.3-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3
|
||||
#spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3
|
||||
|
||||
# ************ Spark UI ****************
|
||||
# Base directory in which Spark events are logged
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-base:0.0.1
|
||||
FROM cyb3rward0g/helk-base:0.0.2
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Kafka."
|
||||
|
||||
|
|
@ -24,10 +24,18 @@ RUN apt-get -qy clean \
|
|||
RUN bash -c 'mkdir -pv /opt/helk/kafka'
|
||||
|
||||
# *********** Install Kafka ***************
|
||||
ENV KAFKA_VERSION=1.1.1
|
||||
ENV KAFKA_VERSION=2.0.0
|
||||
ENV KAFKA_LOGS_PATH=/var/log/kafka
|
||||
ENV KAFKA_CONSOLE_LOG=/var/log/kafka/helk-kafka.log
|
||||
ENV KAFKA_HOME=/opt/helk/kafka/kafka_2.11-${KAFKA_VERSION}
|
||||
ENV KAFKA_HOME=/opt/helk/kafka
|
||||
|
||||
RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ \
|
||||
&& mkdir -v $KAFKA_LOGS_PATH
|
||||
RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ --strip-components=1 \
|
||||
&& mkdir -v $KAFKA_LOGS_PATH
|
||||
|
||||
# ********* Adding Kafka User *************
|
||||
ENV KAFKA_GID=910
|
||||
ENV KAFKA_UID=910
|
||||
ENV KAFKA_USER=kafkauser
|
||||
RUN groupadd -g ${KAFKA_GID} ${KAFKA_USER} \
|
||||
&& useradd -u ${KAFKA_UID} -g ${KAFKA_GID} -d ${KAFKA_HOME} --no-create-home -s /bin/bash ${KAFKA_USER} \
|
||||
&& chown -R ${KAFKA_USER}:${KAFKA_USER} ${KAFKA_HOME} ${KAFKA_LOGS_PATH}
|
||||
|
|
@ -3,21 +3,25 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-kafka-base:1.1.1
|
||||
FROM cyb3rward0g/helk-kafka-base:2.0.0-a
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Kafka Broker."
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
# *********** Configure Kafka Broker ***************
|
||||
ENV KAFKA_SCRIPT=$KAFKA_HOME/bin/kafka-server-start.sh
|
||||
ENV KAFKA_CONFIG=$KAFKA_HOME/config/server.properties
|
||||
|
||||
# *********** Configure Kafka Broker ***************
|
||||
RUN mv $KAFKA_CONFIG ${KAFKA_HOME}/config/backup_server.properties
|
||||
ADD server.properties ${KAFKA_HOME}/config/
|
||||
ADD scripts/kafka-entrypoint.sh /opt/helk/scripts/
|
||||
RUN chmod +x /opt/helk/scripts/kafka-entrypoint.sh
|
||||
COPY server.properties ${KAFKA_HOME}/config/
|
||||
COPY scripts /opt/helk/kafka/scripts
|
||||
RUN chmod +x /opt/helk/kafka/scripts/kafka-entrypoint.sh
|
||||
RUN chmod +x /opt/helk/kafka/scripts/kafka-create-topics.sh
|
||||
|
||||
USER ${KAFKA_USER}
|
||||
|
||||
EXPOSE $KAFKA_BROKER_PORT
|
||||
WORKDIR "/opt/helk/scripts/"
|
||||
ENTRYPOINT ["./kafka-entrypoint.sh"]
|
||||
WORKDIR "/opt/helk/kafka/scripts/"
|
||||
ENTRYPOINT ["./kafka-entrypoint.sh"]
|
||||
CMD ["/bin/bash","-c","$KAFKA_SCRIPT $KAFKA_CONFIG"]
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
#!/bin/bash
|
||||
|
||||
# HELK script: kafka-create-topics.sh
|
||||
# HELK script description: creates kafka topics
|
||||
# HELK build Stage: Alpha
|
||||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
# *********** Configuring Kafka **************
|
||||
if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] No topics will be created"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -z "$REPLICATION_FACTOR" ]]; then
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting replication factor for topics to $REPLICATION_FACTOR"
|
||||
else
|
||||
REPLICATION_FACTOR=1
|
||||
fi
|
||||
|
||||
if [[ ! -z "$ZOOKEEPER_NAME" ]]; then
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Zookeeper name to $ZOOKEEPER_NAME"
|
||||
else
|
||||
ZOOKEEPER_NAME=localhost
|
||||
fi
|
||||
|
||||
# *********** Waiting for Kafka broker to be up ***************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Checking to see if Kafka broker is up..."
|
||||
while [[ "$(curl -sm5 $KAFKA_BROKER_NAME:$KAFKA_BROKER_PORT -o /dev/null; echo $?)" != 56 ]] ; do
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Kafka broker $KAFKA_BROKER_NAME is not available yet"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Kafka is up now..."
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Giving kakfa some time to connect to Zookeeper..."
|
||||
sleep 10
|
||||
|
||||
# *********** Creating Kafka Topics**************
|
||||
declare -a temas=("winlogbeat" "sysmontransformed" "securitytransformed")
|
||||
|
||||
for t in ${temas[@]}; do
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka ${t} Topic.."
|
||||
${KAFKA_HOME}/bin/kafka-topics.sh --create --zookeeper ${ZOOKEEPER_NAME}:2181 --replication-factor ${REPLICATION_FACTOR} --partitions 1 --topic ${t} --if-not-exists
|
||||
done
|
||||
|
||||
wait
|
||||
|
|
@ -18,28 +18,7 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -z "$REPLICATION_FACTOR" ]]; then
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting replication factor for topics to $REPLICATION_FACTOR"
|
||||
else
|
||||
REPLICATION_FACTOR=1
|
||||
fi
|
||||
./kafka-create-topics.sh &
|
||||
unset KAFKA_CREATE_TOPICS
|
||||
|
||||
if [[ ! -z "$ZOOKEEPER_NAME" ]]; then
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Zookeeper name to $ZOOKEEPER_NAME"
|
||||
else
|
||||
ZOOKEEPER_NAME=localhost
|
||||
fi
|
||||
|
||||
# *********** Starting Kafka **************
|
||||
exec $KAFKA_SCRIPT $KAFKA_CONFIG >> $KAFKA_CONSOLE_LOG 2>&1 &
|
||||
sleep 30
|
||||
|
||||
# *********** Creating Kafka Topics**************
|
||||
declare -a temas=("winlogbeat" "sysmontransformed" "securitytransformed")
|
||||
|
||||
for t in ${temas[@]}; do
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka ${t} Topic.."
|
||||
${KAFKA_HOME}/bin/kafka-topics.sh --create --zookeeper ${ZOOKEEPER_NAME}:2181 --replication-factor ${REPLICATION_FACTOR} --partitions 1 --topic ${t} --if-not-exists
|
||||
done
|
||||
|
||||
tail -f $KAFKA_CONSOLE_LOG
|
||||
exec "$@"
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
# HELK script: HELK Kibana Dockerfile
|
||||
# HELK build Stage: Alpha
|
||||
# HELK ELK version: 6.3.2
|
||||
# HELK ELK version: 6.4.0
|
||||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
# References:
|
||||
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
|
||||
|
||||
FROM docker.elastic.co/kibana/kibana:6.3.2
|
||||
FROM docker.elastic.co/kibana/kibana:6.4.0
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Kibana."
|
||||
|
|
@ -6,6 +6,11 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
# *********** Install Plugins *********************
|
||||
#echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Kibana-Canvas.."
|
||||
#NODE_OPTIONS="--max-old-space-size=4096"
|
||||
#kibana-plugin install https://download.elastic.co/kibana/canvas/kibana-canvas-0.1.2174.zip
|
||||
|
||||
# *********** Start Kibana services ***************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
|
||||
until curl -s helk-elasticsearch:9200 -o /dev/null; do
|
||||
|
|
|
|||
|
|
@ -6,10 +6,14 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200
|
||||
# *********** Install Plugins *********************
|
||||
#echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Kibana-Canvas.."
|
||||
#NODE_OPTIONS="--max-old-space-size=4096"
|
||||
#kibana-plugin install https://download.elastic.co/kibana/canvas/kibana-canvas-0.1.2174.zip
|
||||
|
||||
# *********** Check if Elasticsearch is up ***************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.."
|
||||
ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200
|
||||
until curl -s $ELASTICSEARCH_ACCESS -o /dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# HELK script: HELK Logstash Dockerfile
|
||||
# HELK build Stage: Alpha
|
||||
# HELK ELK version: 6.3.2
|
||||
# HELK ELK version: 6.4.0
|
||||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
|
|
@ -8,6 +8,6 @@
|
|||
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
|
||||
# https://github.com/spujadas/elk-docker/blob/master/Dockerfile
|
||||
|
||||
FROM docker.elastic.co/logstash/logstash:6.3.2
|
||||
FROM docker.elastic.co/logstash/logstash:6.4.0
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Logstash."
|
||||
|
|
@ -34,6 +34,9 @@ done
|
|||
# ********** Install Plugin *****************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.."
|
||||
logstash-plugin install logstash-filter-prune
|
||||
# Current FIX to https://discuss.elastic.co/t/kafka-output-plugin-java-lang-long-error/145398
|
||||
logstash-plugin remove logstash-output-kafka
|
||||
logstash-plugin install --version 7.1.1 logstash-output-kafka
|
||||
|
||||
# ********** Starting Logstash *****************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ done
|
|||
# ********** Install Plugin *****************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.."
|
||||
logstash-plugin install logstash-filter-prune
|
||||
# Current FIX to https://discuss.elastic.co/t/kafka-output-plugin-java-lang-long-error/145398
|
||||
logstash-plugin remove logstash-output-kafka
|
||||
logstash-plugin install --version 7.1.1 logstash-output-kafka
|
||||
|
||||
# ********** Starting Logstash *****************
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.."
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html
|
||||
# https://github.com/spujadas/elk-docker/blob/master/Dockerfile
|
||||
|
||||
FROM cyb3rward0g/helk-base:0.0.1
|
||||
FROM cyb3rward0g/helk-base:0.0.2
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Nginx."
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-base:0.0.1
|
||||
FROM cyb3rward0g/helk-base:0.0.2
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for HELK Spark."
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1-a
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for HELK Spark Master."
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1
|
||||
FROM cyb3rward0g/helk-spark-base:2.3.1-a
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for HELK Spark Worker."
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Author: Roberto Rodriguez (@Cyb3rWard0g)
|
||||
# License: GPL-3.0
|
||||
|
||||
FROM cyb3rward0g/helk-kafka-base:1.1.1
|
||||
FROM cyb3rward0g/helk-kafka-base:2.0.0-a
|
||||
LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
|
||||
LABEL description="Dockerfile base for the HELK Kafka Zookeeper."
|
||||
|
||||
|
|
@ -13,7 +13,8 @@ ENV DEBIAN_FRONTEND noninteractive
|
|||
RUN bash -c 'mkdir -pv /opt/helk/zookeeper'
|
||||
|
||||
# *********** ConfigureZookeeper ***************
|
||||
ENV ZOO_CONF_DIR=/opt/helk/zookeeper/conf \
|
||||
ENV ZOO_HOME=/opt/helk/zookeeper \
|
||||
ZOO_CONF_DIR=/opt/helk/zookeeper/conf \
|
||||
ZOO_DATA_DIR=/opt/helk/zookeeper/data \
|
||||
ZOO_DATA_LOG_DIR=/opt/helk/zookeeper/datalog \
|
||||
ZOO_PORT=2181 \
|
||||
|
|
@ -26,11 +27,14 @@ ENV ZOO_CONF_DIR=/opt/helk/zookeeper/conf \
|
|||
|
||||
# *********** Configure zookeeper ***************
|
||||
RUN mkdir -p "$ZOO_CONF_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_LOGS_PATH"
|
||||
ADD scripts/zookeeper-entrypoint.sh /opt/helk/scripts/
|
||||
RUN chmod +x /opt/helk/scripts/zookeeper-entrypoint.sh
|
||||
COPY scripts /opt/helk/zookeeper/scripts
|
||||
RUN chmod +x /opt/helk/zookeeper/scripts/zookeeper-entrypoint.sh
|
||||
|
||||
VOLUME ["$ZOO_DATA_DIR", "$ZOO_DATA_LOG_DIR"]
|
||||
RUN chown -R ${KAFKA_USER}:${KAFKA_USER} ${ZOO_HOME} ${ZOO_LOGS_PATH}
|
||||
|
||||
USER ${KAFKA_USER}
|
||||
|
||||
EXPOSE $ZOO_PORT 2888 3888
|
||||
WORKDIR "/opt/helk/scripts/"
|
||||
ENTRYPOINT ["./zookeeper-entrypoint.sh"]
|
||||
WORKDIR "/opt/helk/zookeeper/scripts/"
|
||||
ENTRYPOINT ["./zookeeper-entrypoint.sh"]
|
||||
CMD ["/bin/bash","-c","/opt/helk/kafka/bin/zookeeper-server-start.sh /opt/helk/zookeeper/conf/zookeeper.properties"]
|
||||
|
|
@ -25,10 +25,4 @@ if [ ! -f $ZOO_CONF_DIR/zookeeper.properties ]; then
|
|||
done
|
||||
fi
|
||||
|
||||
ln -sf /dev/stdout $ZOO_LOGS_FILE
|
||||
|
||||
echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Zookeeper.."
|
||||
KAFKA_SCRIPT_ZOOKEEPER=$KAFKA_HOME/bin/zookeeper-server-start.sh
|
||||
KAFKA_CONFIG_ZOOKEEPER="$ZOO_CONF_DIR/zookeeper.properties"
|
||||
|
||||
exec $KAFKA_SCRIPT_ZOOKEEPER $KAFKA_CONFIG_ZOOKEEPER >> $ZOO_LOGS_FILE 2>&1
|
||||
exec "$@"
|
||||
|
|
@ -29,7 +29,7 @@ check_min_requirements(){
|
|||
if [ "$systemKernel" == "Linux" ]; then
|
||||
AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo)
|
||||
AVAILABLE_DISK=$(df -m | awk '$NF=="/"{printf "%.f\t\t", $4 / 1024}')
|
||||
if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then
|
||||
if [ "${AVAILABLE_MEMORY}" -ge "11" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then
|
||||
echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY"
|
||||
echo "[HELK-INSTALLATION-INFO] Available Disk: $AVAILABLE_DISK"
|
||||
else
|
||||
|
|
@ -68,8 +68,8 @@ install_curl(){
|
|||
install_helk(){
|
||||
# ****** Building & running HELK ***********
|
||||
echo "[HELK-INSTALLATION-INFO] Building & running HELK via docker-compose"
|
||||
echo "[HELK-INSTALLATION-INFO] Using docker-compose-elk-${license_choice}.yml file"
|
||||
docker-compose -f docker-compose-elk-${license_choice}.yml up --build -d >> $LOGFILE 2>&1
|
||||
echo "[HELK-INSTALLATION-INFO] Using docker-compose-helk-elastic-${subscription_choice}.yml file"
|
||||
docker-compose -f docker-compose-helk-elastic-${subscription_choice}.yml up --build -d >> $LOGFILE 2>&1
|
||||
ERROR=$?
|
||||
if [ $ERROR -ne 0 ]; then
|
||||
echoerror "Could not run HELK via docker-compose (Error Code: $ERROR)."
|
||||
|
|
@ -132,19 +132,19 @@ set_helk_ip(){
|
|||
host_ip="${ip_choice:-$host_ip}"
|
||||
}
|
||||
|
||||
set_helk_license(){
|
||||
# *********** Accepting Defaults or Allowing user to set HELK License ***************
|
||||
local license_input
|
||||
read -t 30 -p "[HELK-INSTALLATION-INFO] Set HELK License. Default value is basic: " -e -i "basic" license_input
|
||||
license_choice=${license_input:-"basic"}
|
||||
# *********** Validating License Input ***************
|
||||
case $license_choice in
|
||||
set_helk_subscription(){
|
||||
# *********** Accepting Defaults or Allowing user to set HELK IP ***************
|
||||
local subscription_input
|
||||
read -t 30 -p "[HELK-INSTALLATION-INFO] Set HELK elastic subscription (basic or trial). Default value is basic: " -e -i "basic" subscription_input
|
||||
subscription_choice=${subscription_input:-"basic"}
|
||||
# *********** Validating subscription Input ***************
|
||||
case $subscription_choice in
|
||||
basic)
|
||||
;;
|
||||
trial)
|
||||
;;
|
||||
*)
|
||||
echo "[HELK-INSTALLATION-ERROR] Not a valid license. Valid Options: basic or trial"
|
||||
echo "[HELK-INSTALLATION-ERROR] Not a valid subscription. Valid Options: basic or trial"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -152,7 +152,7 @@ set_helk_license(){
|
|||
|
||||
prepare_helk(){
|
||||
echo "[HELK-INSTALLATION-INFO] HELK IP set to ${host_ip}"
|
||||
echo "[HELK-INSTALLATION-INFO] HELK License set to ${license_choice}"
|
||||
echo "[HELK-INSTALLATION-INFO] HELK elastic subscription set to ${subscription_choice}"
|
||||
if [ "$systemKernel" == "Linux" ]; then
|
||||
# Reference: https://get.docker.com/
|
||||
echo "[HELK-INSTALLATION-INFO] Checking distribution list and version"
|
||||
|
|
@ -210,7 +210,17 @@ prepare_helk(){
|
|||
# *********** Check if docker is installed ***************
|
||||
if [ -x "$(command -v docker)" ]; then
|
||||
echo "[HELK-INSTALLATION-INFO] Docker already installed"
|
||||
|
||||
echo "[HELK-INSTALLATION-INFO] Making sure you assigned enough disk space to the current Docker base directory"
|
||||
AVAILABLE_DOCKER_DISK=$(df -m $(docker info --format '{{.DockerRootDir}}') | awk '$1 ~ /\//{printf "%.f\t\t", $4 / 1024}')
|
||||
if [ "${AVAILABLE_DOCKER_DISK}" -ge "30" ]; then
|
||||
echo "[HELK-INSTALLATION-INFO] Available Docker Disk: $AVAILABLE_DOCKER_DISK"
|
||||
else
|
||||
echo "[HELK-INSTALLATION-ERROR] YOU DO NOT HAVE ENOUGH DOCKER DISK SPACE ASSIGNED"
|
||||
echo "[HELK-INSTALLATION-ERROR] Available Docker Disk: $AVAILABLE_DOCKER_DISK"
|
||||
echo "[HELK-INSTALLATION-ERROR] Check the requirements section in our installation Wiki"
|
||||
echo "[HELK-INSTALLATION-ERROR] Installation Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/Installation"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[HELK-INSTALLATION-INFO] Docker is not installed"
|
||||
|
||||
|
|
@ -251,7 +261,7 @@ prepare_helk(){
|
|||
fi
|
||||
echo "[HELK-INSTALLATION-INFO] Setting KAFKA ADVERTISED_LISTENER value..."
|
||||
# ****** Setting KAFKA ADVERTISED_LISTENER environment variable ***********
|
||||
sed -i "s/ADVERTISED_LISTENER=HOSTIP/ADVERTISED_LISTENER=$host_ip/g" docker-compose-elk-${license_choice}.yml
|
||||
sed -i "s/ADVERTISED_LISTENER=HOSTIP/ADVERTISED_LISTENER=$host_ip/g" docker-compose-helk-elastic-${subscription_choice}.yml
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -262,8 +272,8 @@ show_banner(){
|
|||
echo "** HELK - THE HUNTING ELK **"
|
||||
echo "** **"
|
||||
echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **"
|
||||
echo "** HELK build version: v0.1.2-alpha08062018 **"
|
||||
echo "** HELK ELK version: 6.3.2 **"
|
||||
echo "** HELK build version: v0.1.3-alpha08242018 **"
|
||||
echo "** HELK ELK version: 6.4.0 **"
|
||||
echo "** License: GPL-3.0 **"
|
||||
echo "**********************************************"
|
||||
echo " "
|
||||
|
|
@ -283,6 +293,7 @@ show_final_information(){
|
|||
echo "HELK JUPYTERHUB URL: http://${host_ip}/jupyter"
|
||||
echo "HELK JUPYTERHUB USER:PWD : hunter1:hunter1P@ssw0rd!"
|
||||
echo "HELK JUPYTERHUB USER:PWD : hunter2:hunter2P@ssw0rd!"
|
||||
echo "HELK JUPYTERHUB USER:PWD : hunter3:hunter3P@ssw0rd!"
|
||||
echo "HELK SPARK MASTER UI: http://${host_ip}:8080"
|
||||
echo " "
|
||||
echo "IT IS HUNTING SEASON!!!!!"
|
||||
|
|
@ -296,7 +307,7 @@ manual_install(){
|
|||
check_min_requirements
|
||||
get_host_ip
|
||||
set_helk_ip
|
||||
set_helk_license
|
||||
set_helk_subscription
|
||||
prepare_helk
|
||||
install_helk
|
||||
sleep 180
|
||||
|
|
@ -317,13 +328,13 @@ usage(){
|
|||
echo "Usage: $0 [option...]" >&2
|
||||
echo
|
||||
echo " -i set HELKs IP address"
|
||||
echo " -l set HELKs License (basic or trial)"
|
||||
echo " -l set HELKs subscription (basic or trial)"
|
||||
echo " -q quiet -> not output to the console"
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo " $0 Install HELK manually"
|
||||
echo " $0 -i 192.168.64.131 -l basic Install HELK with an IP address set and basic License"
|
||||
echo " $0 -i 192.168.64.131 -l trial -q Install HELK with an IP address set and trial License without sending output to the console"
|
||||
echo " $0 -i 192.168.64.131 -l basic Install HELK with an IP address set and basic subscription"
|
||||
echo " $0 -i 192.168.64.131 -l trial -q Install HELK with an IP address set and trial subscription without sending output to the console"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
|
@ -339,7 +350,7 @@ while getopts ":i:l:q" opt; do
|
|||
quiet="TRUE"
|
||||
;;
|
||||
l )
|
||||
license_choice=$OPTARG
|
||||
subscription_choice=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "[HELK-INSTALLATION-ERROR] Invalid option: $OPTARG" 1>&2
|
||||
|
|
@ -356,7 +367,7 @@ if [ $# -gt 0 ]; then
|
|||
echo "[HELK-INSTALLATION-ERROR] Invalid option"
|
||||
usage
|
||||
fi
|
||||
if [ -z "$host_ip" ] && [ -z "$quiet" ] && [ -z "$license_choice" ]; then
|
||||
if [ -z "$host_ip" ] && [ -z "$quiet" ] && [ -z "$subscription_choice" ]; then
|
||||
manual_install
|
||||
else
|
||||
if [[ "$host_ip" =~ ^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$ ]]; then
|
||||
|
|
@ -366,14 +377,14 @@ else
|
|||
usage
|
||||
fi
|
||||
done
|
||||
# *********** Validating License Input ***************
|
||||
case $license_choice in
|
||||
# *********** Validating subscription Input ***************
|
||||
case $subscription_choice in
|
||||
basic)
|
||||
;;
|
||||
trial)
|
||||
;;
|
||||
*)
|
||||
echo "[HELK-INSTALLATION-ERROR] Not a valid license. Valid Options: basic or trial"
|
||||
echo "[HELK-INSTALLATION-ERROR] Not a valid subscription. Valid Options: basic or trial"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue