Updating pipeline

+ added new topic to replace winlogbeat in future updates
+ updated nxlog mordor to test raw events

docker/helk-kibana-analysis-alert-basic.yml

@@ -135,7 +135,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-analysis-alert-trial.yml

@@ -138,7 +138,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-analysis-basic.yml

@@ -110,7 +110,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-analysis-trial.yml

@@ -138,7 +138,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-notebook-analysis-alert-basic.yml

@@ -135,7 +135,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-notebook-analysis-alert-trial.yml

@@ -139,7 +139,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1g -Xms1g
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-notebook-analysis-basic.yml

@@ -135,7 +135,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1G -Xms1G
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-kibana-notebook-analysis-trial.yml

@@ -139,7 +139,7 @@ services:
       REPLICATION_FACTOR: 1
       ADVERTISED_LISTENER: ${ADVERTISED_LISTENER}
       ZOOKEEPER_NAME: helk-zookeeper
-      KAFKA_CREATE_TOPICS: winlogbeat, SYSMON_JOIN, filebeat, mordor
+      KAFKA_CREATE_TOPICS: winlogbeat, winevent, SYSMON_JOIN, filebeat
       KAFKA_HEAP_OPTS: -Xmx1g -Xms1g
       LOG_RETENTION_HOURS: 4
     ports:

docker/helk-logstash/mordor_pipeline/0001-nxlog-input.conf

@@ -6,7 +6,5 @@
 input {
    tcp {
       port => 3515
-      type => "nxlog-mordor"
-      #codec => json { charset => "CP1252" }
    }
 }

docker/helk-logstash/mordor_pipeline/0003-nxlog-to-json.conf

@@ -4,12 +4,10 @@
 # License: GPL-3.0
 
 filter {
-  if [type] == "nxlog-mordor" {
-    json {
-      source => "message"
-      tag_on_failure => [ "_jsonparsefailure", "_parsefailure", "_jsonparsefailure_0301" ]
-      remove_field => [ "message" ]
-      add_field => { "z_logstash_pipeline" => "json-0003-001" }
-    }
+  json {
+    source => "message"
+    tag_on_failure => [ "_jsonparsefailure", "_parsefailure", "_jsonparsefailure_0301" ]
+    remove_field => [ "Message" ]
+    add_tag => [ "mordorDataset" ]
   }
 }

docker/helk-logstash/mordor_pipeline/0004-nxlog-output.conf

@@ -4,11 +4,9 @@
 # License: GPL-3.0
 
 output {
-   if [type] == "nxlog-mordor" {
-     kafka {
-       bootstrap_servers => "helk-kafka-broker:9092"
-       codec => "json"
-       topic_id => "mordor"
-     }
-   }
+  kafka {
+    bootstrap_servers => "helk-kafka-broker:9092"
+    codec => "json"
+    topic_id => "winevent"
+  }
 }

docker/helk-logstash/pipeline/0002-kafka-input.conf

@@ -6,7 +6,7 @@
 input {
   kafka {
     bootstrap_servers => "helk-kafka-broker:9092"
-    topics => ["winlogbeat", "SYSMON_JOIN","filebeat"]
+    topics => ["winlogbeat","winevent","SYSMON_JOIN","filebeat"]
     decorate_events => true
     codec => "json"
     auto_offset_reset => "latest"