diff --git a/docker/helk-logstash/output_templates/11-indexme.json b/docker/helk-logstash/output_templates/11-indexme.json index ba1ddbf..db7356e 100644 --- a/docker/helk-logstash/output_templates/11-indexme.json +++ b/docker/helk-logstash/output_templates/11-indexme.json @@ -1,12 +1,12 @@ { "order": 11, "index_patterns": [ "indexme-*" ], - "version": 2019021901, + "version": 2019052001, "settings": { "index": { "mapping": { "ignore_malformed": true, - "total_fields.limit": "1000", + "total_fields.limit": "5000", "coerce": true }, "store": { diff --git a/docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf b/docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf index 11fc2f4..7aeb42d 100644 --- a/docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf +++ b/docker/helk-logstash/pipeline/1010-winevent-winlogbeats-filter.conf @@ -27,6 +27,7 @@ filter { "beat_version" => "%{[beat][version]}" "beat_name" => "%{[beat][name]}" "z_logstash_pipeline" => "winlogbeat_6-field_nest_cleanup" + "[@metadata][helk_parsed]" => "yes" } remove_field => [ "[beat]" ] } @@ -65,7 +66,10 @@ filter { event.remove("[winlog][user]") ' tag_on_exception => "ruby_exception_winlogbeat_7-cleanup" - add_field => { "z_logstash_pipeline" => "winlogbeat_7-field_nest_cleanup" } + add_field => { + "z_logstash_pipeline" => "winlogbeat_7-field_nest_cleanup" + "[@metadata][helk_parsed]" => "yes" + } } # Also, for continuity copy the new fields for Winlogbeat 7 back to the original field names (Winlogbeat 6.x and before). However, lets keep them - for future or anyone else doing something different. mutate { diff --git a/docker/helk-logstash/pipeline/1050-nxlog-winevent-to-winlogbeats-merge-filter.conf b/docker/helk-logstash/pipeline/1050-nxlog-winevent-to-winlogbeats-merge-filter.conf index 913dad6..228099a 100644 --- a/docker/helk-logstash/pipeline/1050-nxlog-winevent-to-winlogbeats-merge-filter.conf +++ b/docker/helk-logstash/pipeline/1050-nxlog-winevent-to-winlogbeats-merge-filter.conf @@ -26,6 +26,7 @@ filter { add_field => { "type" => "wineventlog" "z_logstash_pipeline" => "mutate-1050-0001" + "[@metadata][helk_parsed]" => "yes" } rename => { "Message" => "message" } } diff --git a/docker/helk-logstash/pipeline/1216-attack-filter.conf b/docker/helk-logstash/pipeline/1216-attack-filter.conf index 74ce662..3aca073 100644 --- a/docker/helk-logstash/pipeline/1216-attack-filter.conf +++ b/docker/helk-logstash/pipeline/1216-attack-filter.conf @@ -17,6 +17,7 @@ filter { 'remote_support','system_requirements','contributors','url','tactic','platform','data_sources', 'permissions_required','Validation' ] + add_field => { "[@metadata][helk_parsed]" => "yes" } } } } \ No newline at end of file diff --git a/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf b/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf index 37eb02e..6f51fa2 100644 --- a/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf +++ b/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [log_name] == "Microsoft-Windows-Sysmon/Operational"{ + if [log_name] == "Microsoft-Windows-Sysmon/Operational" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-sysmon-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9951-winevent-security-output.conf b/docker/helk-logstash/pipeline/9951-winevent-security-output.conf index 3fca7dc..b556d97 100644 --- a/docker/helk-logstash/pipeline/9951-winevent-security-output.conf +++ b/docker/helk-logstash/pipeline/9951-winevent-security-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [log_name] == "Security"{ + if [log_name] == "Security" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-security-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9952-winevent-system-output.conf b/docker/helk-logstash/pipeline/9952-winevent-system-output.conf index d9d8531..1261aa1 100644 --- a/docker/helk-logstash/pipeline/9952-winevent-system-output.conf +++ b/docker/helk-logstash/pipeline/9952-winevent-system-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [log_name] == "System"{ + if [log_name] == "System" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-system-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9953-winevent-application-output.conf b/docker/helk-logstash/pipeline/9953-winevent-application-output.conf index 09bd5b4..900d0b3 100644 --- a/docker/helk-logstash/pipeline/9953-winevent-application-output.conf +++ b/docker/helk-logstash/pipeline/9953-winevent-application-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [log_name] == "Application"{ + if [log_name] == "Application" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-application-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf b/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf index f8919e5..de5c6db 100644 --- a/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf +++ b/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{ + if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-powershell-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf b/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf index 37104a4..f07f4ac 100644 --- a/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf +++ b/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf @@ -4,7 +4,7 @@ # License: GPL-3.0 output { - if [log_name] == "Microsoft-Windows-WMI-Activity/Operational"{ + if [log_name] == "Microsoft-Windows-WMI-Activity/Operational" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-wmiactivity-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9956-attack-output.conf b/docker/helk-logstash/pipeline/9956-attack-output.conf index 459ecd3..c8e9683 100644 --- a/docker/helk-logstash/pipeline/9956-attack-output.conf +++ b/docker/helk-logstash/pipeline/9956-attack-output.conf @@ -5,7 +5,7 @@ # License: GPL-3.0 output { - if "attack" in [tags]{ + if "attack" in [tags] and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "mitre-attack-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9959-winevent-codeintegrity-output.conf b/docker/helk-logstash/pipeline/9959-winevent-codeintegrity-output.conf index 0056baa..e08e295 100644 --- a/docker/helk-logstash/pipeline/9959-winevent-codeintegrity-output.conf +++ b/docker/helk-logstash/pipeline/9959-winevent-codeintegrity-output.conf @@ -1,5 +1,5 @@ output { - if [log_name] == "Microsoft-Windows-CodeIntegrity/Operational"{ + if [log_name] == "Microsoft-Windows-CodeIntegrity/Operational" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-codeintegrity-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9960-winevent-bits-output.conf b/docker/helk-logstash/pipeline/9960-winevent-bits-output.conf index 8160497..3b0d626 100644 --- a/docker/helk-logstash/pipeline/9960-winevent-bits-output.conf +++ b/docker/helk-logstash/pipeline/9960-winevent-bits-output.conf @@ -1,5 +1,5 @@ output { - if [log_name] == "Microsoft-Windows-Bits-Client/Operational"{ + if [log_name] == "Microsoft-Windows-Bits-Client/Operational" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-bits-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9961-winevent-dns-client-output.conf b/docker/helk-logstash/pipeline/9961-winevent-dns-client-output.conf index 0df7e63..4d84c8f 100644 --- a/docker/helk-logstash/pipeline/9961-winevent-dns-client-output.conf +++ b/docker/helk-logstash/pipeline/9961-winevent-dns-client-output.conf @@ -1,5 +1,5 @@ output { - if [log_name] == "Microsoft-Windows-DNS-Client/Operational"{ + if [log_name] == "Microsoft-Windows-DNS-Client/Operational" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-dns-client-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9962-winevent-firewall-advanced-output.conf b/docker/helk-logstash/pipeline/9962-winevent-firewall-advanced-output.conf index 6693821..6dc1a08 100644 --- a/docker/helk-logstash/pipeline/9962-winevent-firewall-advanced-output.conf +++ b/docker/helk-logstash/pipeline/9962-winevent-firewall-advanced-output.conf @@ -1,5 +1,5 @@ output { - if [log_name] == "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall"{ + if [log_name] == "Microsoft-Windows-Windows Firewall With Advanced Security/Firewall" and [@metadata][helk_parsed] == "yes" { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-firewall-advanced-%{+YYYY.MM.dd}" diff --git a/docker/helk-logstash/pipeline/9998-catch_all-output.conf b/docker/helk-logstash/pipeline/9998-catch_all-output.conf new file mode 100644 index 0000000..954fa48 --- /dev/null +++ b/docker/helk-logstash/pipeline/9998-catch_all-output.conf @@ -0,0 +1,11 @@ +output { + if [@metadata][helk_parsed] != "yes" and [source] != "/var/log/osquery/osqueryd.results.log" and [@metadata][kafka][topic] != "SYSMON_JOIN" { + elasticsearch { + hosts => ["helk-elasticsearch:9200"] + index => "indexme-%{+YYYY.MM.dd}" + # document_id => "%{[@metadata][log_hash]}" + user => 'elastic' + #password => 'elasticpassword' + } + } +} \ No newline at end of file