support for Winlogbeat 7 and keep backwards compatibility for Winlogbeat 6

docker/helk-logstash/pipeline/0099-all-fingerprint-hash-filter.conf

@@ -7,7 +7,8 @@
 
 filter {
 
-  # Perform hashing on winlogbeat differently than other logs
+  ## Perform hashing on winlogbeat differently than other logs
+  # Winlogbeat 6.x
   if [type] == "wineventlog" and [beat] {
     fingerprint {
       source => [
@@ -23,6 +24,22 @@ filter {
       add_field => { "z_logstash_pipeline" => "fingerprint-0099-001" }
     }
   }
+  # Winlogbeat 7.x
+  if [winlog][api] == "wineventlog" and [agent][type] == "winlogbeat" {
+    fingerprint {
+      source => [
+          "message",
+          "[winlog][computer_name]",
+          "@timestamp",
+          "[winlog][channel]",
+          "[winlog][event_id]"
+     ]
+      concatenate_sources => true
+      target => "[@metadata][log_hash]"
+      method => "SHA1"
+      add_field => { "z_logstash_pipeline" => "fingerprint-winlogbeats7" }
+    }
+  }
 
   # Perform hashing on NXLog differently than other logs
   else if [type] == "nxlog-winevent" {
@@ -68,8 +85,9 @@ filter {
     }
   }
 
-  # Scenario of no message field for, create custom one concatenating some values to guarantee unique fingerprint
+  ## Scenario of no message field for, create custom one concatenating some values to guarantee unique fingerprint
   #TONOTE: can use this value in z_logstash_pipeline to see if this event is hit
+  # 6.x beats
   else if [beat] {
     # Use this custom for this event
     mutate { add_field => { "meta_log_tags" => "warning missing message field" } }
@@ -80,5 +98,16 @@ filter {
       add_field => { "z_logstash_pipeline" => "fingerprint-0099-006" }
     }
   }
+  # 7.x beats
+  else if [agent] {
+    # Use this custom for this event
+    mutate { add_field => { "meta_log_tags" => "warning missing message field" } }
+    fingerprint {
+      concatenate_all_fields => true
+      target => "[@metadata][log_hash]"
+      method => "SHA1"
+      add_field => { "z_logstash_pipeline" => "fingerprint-beats7-missing-message-field" }
+    }
+  }
 
 }

docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf

@@ -4,9 +4,9 @@
 # License: GPL-3.0
 
 filter {
-  # Use the following to get rid of the prepended "event_data" nest that (elastic) winlogbeats adds to windows logs
+  ## Use the following to get rid of the prepended fields that (elastic) winlogbeats adds
+  # Winlogbeat 6.x
   if [type] == "wineventlog" and [beat] {
-    mutate { add_field => { "z_logstash_pipeline" => "1010" } }
     ruby {
       code => "
         eventdata = event.get('event_data')
@@ -21,22 +21,66 @@ filter {
         # Finally remove the nest completely
         event.remove('event_data')
       "
-      tag_on_exception =>  "_rubyexception_1010"
-      #code => "
-      #  event.get('event_data').each {|k, v|
-      #    event.set(k, v)
-      #  }
-      #  event.remove('event_data')
-      #"
-      #tag_on_exception =>  "_rubyexception_1010"
-    }
-    mutate {
+      tag_on_exception =>  "winlogbeat_6_cleanup"
       add_field => {
         "beat_hostname" => "%{[beat][hostname]}"
         "beat_version" => "%{[beat][version]}"
         "beat_name" => "%{[beat][name]}"
+        "z_logstash_pipeline" => "winlogbeat_6-field_nest_cleanup"
       }
       remove_field => [ "[beat]" ]
     }
   }
+  # Winlogbeat 7.x
+  else if [agent][type] == "winlogbeat" {
+    ruby {
+      code => "
+        eventdata = event.get('[winlog][event_data]')
+        # Sometimes does not exist, so check that first -- then move the nests
+        if !eventdata.nil?
+          eventdata.each {|k, v|
+            if eventdata.to_s != '(NULL)'
+              event.set(k, v)
+            end
+          }
+        end
+        # Finally remove the nest completely
+        event.remove('[winlog][event_data]')
+      "
+      tag_on_exception =>  "winlogbeat_7-cleanup"
+      add_field => { "z_logstash_pipeline" => "winlogbeat_7-field_nest_cleanup" }
+      remove_field => [ "[beat]" ]
+    }
+    # Also, for continuity copy the new fields for Winlogbeat 7 back to the original field names (Winlogbeat 6.x and before). However, lets keep them - for future or anyone else doing something different.
+    mutate {
+      copy => {
+        "[agent][hostname]" => "beat_hostname"
+        "[agent][name]" => "beat_name"
+        "[agent][version]" => "beat_version"
+        "[event][timezone]" => "beat_timezone"
+        "[log][level]" => "level"
+        "[error][message]" => "message_error"
+        "[event][original]" => "xml"
+        "[process][executable]" => "[process][exe]"
+        "[winlog][activity_id]" => "activity_id"
+        "[winlog][api]" => "type"
+        "[winlog][channel]" => "log_name"
+        "[winlog][computer_name]" => "computer_name"
+        "[winlog][event_id]" => "event_id"
+        "[winlog][keywords]" => "keywords"
+        "[winlog][provider_guid]" => "provider_guid"
+        "[winlog][provider_name]" => "source_name"
+        "[winlog][process][pid]" => "process_id"
+        "[winlog][process][thread][id]" => "thread_id"
+        "[winlog][opcode]" => "opcode"
+        "[winlog][record_id]" => "record_number"
+        "[winlog][task]" => "task"
+        "[winlog][user][domain]" => "[user][domain]"
+        "[winlog][user][identifier]" => "[user][identifier]"
+        "[winlog][user][type]" => "[user][type]"
+        "[winlog][version]" => "version"
+      }
+      add_field => { "z_logstash_pipeline" => "winlogbeat_7-copy_to_originals" }
+    }
+  }
 }