implement catchall index

docker/helk-logstash/output_templates/11-indexme.json

@@ -1,12 +1,12 @@
 {
   "order": 11,
   "index_patterns": [ "indexme-*" ],
-  "version": 2019021901,
+  "version": 2019052001,
   "settings": {
     "index": {
       "mapping": {
         "ignore_malformed": true,
-        "total_fields.limit": "1000",
+        "total_fields.limit": "5000",
         "coerce": true
       },
       "store": {

docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf

@@ -27,6 +27,7 @@ filter {
         "beat_version" => "%{[beat][version]}"
         "beat_name" => "%{[beat][name]}"
         "z_logstash_pipeline" => "winlogbeat_6-field_nest_cleanup"
+        "[@metadata][helk_parsed]" => "yes"
       }
       remove_field => [ "[beat]" ]
     }
@@ -65,7 +66,10 @@ filter {
         event.remove("[winlog][user]")
       '
       tag_on_exception =>  "ruby_exception_winlogbeat_7-cleanup"
-      add_field => { "z_logstash_pipeline" => "winlogbeat_7-field_nest_cleanup" }
+      add_field => {
+        "z_logstash_pipeline" => "winlogbeat_7-field_nest_cleanup"
+        "[@metadata][helk_parsed]" => "yes"
+      }
     }
     # Also, for continuity copy the new fields for Winlogbeat 7 back to the original field names (Winlogbeat 6.x and before). However, lets keep them - for future or anyone else doing something different.
     mutate {

docker/helk-logstash/pipeline/1050-nxlog-winevent-to-winlogbeats-merge-filter.conf

@@ -26,6 +26,7 @@ filter {
       add_field => {
         "type" => "wineventlog"
         "z_logstash_pipeline" => "mutate-1050-0001"
+        "[@metadata][helk_parsed]" => "yes"
       }
       rename => { "Message" => "message" }
     }

docker/helk-logstash/pipeline/1216-attack-filter.conf

@@ -17,6 +17,7 @@ filter {
         'remote_support','system_requirements','contributors','url','tactic','platform','data_sources',
         'permissions_required','Validation'
         ]
+        add_field => { "[@metadata][helk_parsed]" => "yes" }
     }
   }
 }

docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [log_name] == "Microsoft-Windows-Sysmon/Operational"{
+  if [log_name] == "Microsoft-Windows-Sysmon/Operational" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-sysmon-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9951-winevent-security-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [log_name] == "Security"{
+  if [log_name] == "Security" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-security-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9952-winevent-system-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [log_name] == "System"{
+  if [log_name] == "System" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-system-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9953-winevent-application-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [log_name] == "Application"{
+  if [log_name] == "Application" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-application-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{
+  if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-powershell-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf

@@ -4,7 +4,7 @@
 # License: GPL-3.0
 
 output {
-  if [log_name] == "Microsoft-Windows-WMI-Activity/Operational"{
+  if [log_name] == "Microsoft-Windows-WMI-Activity/Operational" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-wmiactivity-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9956-attack-output.conf

@@ -5,7 +5,7 @@
 # License: GPL-3.0
 
 output {
-  if "attack" in [tags]{
+  if "attack" in [tags] and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "mitre-attack-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9959-winevent-codeintegrity-output.conf

@@ -1,5 +1,5 @@
 output {
-  if [log_name] == "Microsoft-Windows-CodeIntegrity/Operational"{
+  if [log_name] == "Microsoft-Windows-CodeIntegrity/Operational" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-codeintegrity-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9960-winevent-bits-output.conf

@@ -1,5 +1,5 @@
 output {
-  if [log_name] == "Microsoft-Windows-Bits-Client/Operational"{
+  if [log_name] == "Microsoft-Windows-Bits-Client/Operational" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-bits-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9961-winevent-dns-client-output.conf

@@ -1,5 +1,5 @@
 output {
-  if [log_name] == "Microsoft-Windows-DNS-Client/Operational"{
+  if [log_name] == "Microsoft-Windows-DNS-Client/Operational" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-dns-client-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9962-winevent-firewall-advanced-output.conf

@@ -1,5 +1,5 @@
 output {
-  if [log_name] == "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall"{
+  if [log_name] == "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall" and [@metadata][helk_parsed] == "yes" {
     elasticsearch {
       hosts => ["helk-elasticsearch:9200"]
       index => "logs-endpoint-winevent-firewall-advanced-%{+YYYY.MM.dd}"

docker/helk-logstash/pipeline/9998-catch_all-output.conf

@@ -0,0 +1,11 @@
+output {
+  if [@metadata][helk_parsed] != "yes" and [source] != "/var/log/osquery/osqueryd.results.log" and [@metadata][kafka][topic] != "SYSMON_JOIN" {
+    elasticsearch {
+      hosts => ["helk-elasticsearch:9200"]
+      index => "indexme-%{+YYYY.MM.dd}"
+      # document_id => "%{[@metadata][log_hash]}"
+      user => 'elastic'
+      #password => 'elasticpassword'
+    }
+  }
+}