infosecn1nja
/
HELK
mirror of https://github.com/infosecn1nja/HELK.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updating pipeline 

+ added new topic to replace winlogbeat in future updates
+ updated nxlog mordor to test raw events
neu5ron-patch-1
Cyb3rWard0g 2020-01-05 17:44:25 -05:00
parent 060fdf7a2a
commit c6c272c2e6
12 changed files with 19 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/helk-kibana-analysis-alert-basic.yml
View File

@ -135,7 +135,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-analysis-alert-trial.yml
View File

@ -138,7 +138,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-analysis-basic.yml
View File

@ -110,7 +110,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-analysis-trial.yml
View File

@ -138,7 +138,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-notebook-analysis-alert-basic.yml
View File

@ -135,7 +135,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-notebook-analysis-alert-trial.yml
View File

@ -139,7 +139,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1g -Xms1g KAFKA_HEAP_OPTS: -Xmx1g -Xms1g
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-notebook-analysis-basic.yml
View File

@ -135,7 +135,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1G -Xms1G KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-kibana-notebook-analysis-trial.yml
View File

@ -139,7 +139,7 @@ services:
REPLICATION_FACTOR: 1 REPLICATION_FACTOR: 1
ADVERTISED_LISTENER: ${ADVERTISED_LISTENER} ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
ZOOKEEPER_NAME: helk-zookeeper ZOOKEEPER_NAME: helk-zookeeper
KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
KAFKA_HEAP_OPTS: -Xmx1g -Xms1g KAFKA_HEAP_OPTS: -Xmx1g -Xms1g
LOG_RETENTION_HOURS: 4 LOG_RETENTION_HOURS: 4
ports: ports:

2
docker/helk-logstash/mordor_pipeline/0001-nxlog-input.conf
View File

@ -6,7 +6,5 @@
input { input {
tcp { tcp {
port => 3515 port => 3515
type => "nxlog-mordor"
#codec => json { charset => "CP1252" }
} }
} }

12
docker/helk-logstash/mordor_pipeline/0003-nxlog-to-json.conf
View File

@ -4,12 +4,10 @@
# License: GPL-3.0 # License: GPL-3.0
filter { filter {
if [type] == "nxlog-mordor" { json {
json { source => "message"
source => "message" tag_on_failure => [ "_jsonparsefailure", "_parsefailure", "_jsonparsefailure_0301" ]
tag_on_failure => [ "_jsonparsefailure", "_parsefailure", "_jsonparsefailure_0301" ] remove_field => [ "Message" ]
remove_field => [ "message" ] add_tag => [ "mordorDataset" ]
add_field => { "z_logstash_pipeline" => "json-0003-001" }
}
} }
} }

12
docker/helk-logstash/mordor_pipeline/0004-nxlog-output.conf
View File

@ -4,11 +4,9 @@
# License: GPL-3.0 # License: GPL-3.0
output { output {
if [type] == "nxlog-mordor" { kafka {
kafka { bootstrap_servers => "helk-kafka-broker:9092"
bootstrap_servers => "helk-kafka-broker:9092" codec => "json"
codec => "json" topic_id => "winevent"
topic_id => "mordor" }
}
}
} }

2
docker/helk-logstash/pipeline/0002-kafka-input.conf
View File

@ -6,7 +6,7 @@
input { input {
kafka { kafka {
bootstrap_servers => "helk-kafka-broker:9092" bootstrap_servers => "helk-kafka-broker:9092"
topics => ["winlogbeat", "SYSMON_JOIN","filebeat"] topics => ["winlogbeat","winevent","SYSMON_JOIN","filebeat"]
decorate_events => true decorate_events => true
codec => "json" codec => "json"
auto_offset_reset => "latest" auto_offset_reset => "latest"