Merge branch 'master' of development branch

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7-alpine
 
-LABEL version="1.2.4"
+LABEL version="1.2.5"
 
 # update repository and install Linux packages
 RUN apk update && \

README.md

@@ -1,7 +1,7 @@
 <img src="https://github.com/rabobank-cdc/DeTTECT/wiki/images/logo.png" alt="DeTT&CT" width=30% height=30%>
 
 #### Detect Tactics, Techniques & Combat Threats
-Latest version: [1.2.4](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-124)
+Latest version: [1.2.5](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-125)
 
 To get started with DeTT&CT, check out this [page](https://github.com/rabobank-cdc/DeTTECT/wiki/Getting-started), our [talk](https://www.youtube.com/watch?v=_kWpekkhomU) at hack.lu 2019 and our blog on:
 - [mbsecure.nl/blog/2019/5/dettact-mapping-your-blue-team-to-mitre-attack](https://www.mbsecure.nl/blog/2019/5/dettact-mapping-your-blue-team-to-mitre-attack) or

constants.py

@@ -2,7 +2,7 @@ import re
 
 APP_NAME = 'DeTT&CT'
 APP_DESC = 'Detect Tactics, Techniques & Combat Threats'
-VERSION = '1.2.4'
+VERSION = '1.2.5'
 
 EXPIRE_TIME = 60 * 60 * 24
 

data_source_mapping.py

@@ -495,7 +495,7 @@ def generate_technique_administration_file(filename, write_file=True):
     # Score visibility based on the number of available data sources and the exceptions
     for t in techniques:
         platforms = t.get('x_mitre_platforms', None)
-        if len(set(platforms).intersection(set(platform))) > 0:
+        if platform == 'all' or len(set(platforms).intersection(set(platform))) > 0:
             # not every technique has data source listed
             if 'x_mitre_data_sources' in t:
                 total_ds_count = len(t['x_mitre_data_sources'])
@@ -536,7 +536,7 @@ def generate_technique_administration_file(filename, write_file=True):
         # remove the single quotes from the date
         yaml_file_lines = fix_date_and_remove_null(file_lines, today, input_type='list')
 
-        output_filename = get_non_existing_filename('output/techniques-administration-' + normalize_name_to_filename(name+'-'+'-'.join(platform)), 'yaml')
+        output_filename = get_non_existing_filename('output/techniques-administration-' + normalize_name_to_filename(name +'-' +platform_to_filename(platform)), 'yaml')
         with open(output_filename, 'w') as f:
             f.writelines(yaml_file_lines)
         print("File written:   " + output_filename)

dettect.py

@@ -185,7 +185,7 @@ def _menu(menu_parser):
             file_ds = args.file_ds
 
             if args.search:
-                file_ds = search(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, args.search)
+                file_ds = data_source_search(args.file_ds, args.search)
                 if not file_ds:
                     quit()  # something went wrong in executing the search or 0 results where returned
             if args.update and check_file(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION, args.health):

eql_yaml.py

@@ -172,7 +172,8 @@ def _events_to_yaml(query_results, obj_type):
             # when using an EQL query that does not result in a dict having valid YAML 'data_source' objects.
             return None
 
-        if check_health_data_sources(None, {'data_sources': query_results}, health_is_called=False, no_print=True):
+        if check_health_data_sources(None, {'data_sources': query_results}, health_is_called=False, no_print=True,
+                                     skip_platform=True):
             print(EQL_INVALID_RESULT_DS)
             pprint(query_results)
             return None
@@ -426,33 +427,26 @@ def techniques_search(filename, query_visibility=None, query_detection=None, inc
     return yaml_content
 
 
-def search(filename, file_type, query='', include_all_score_objs=False):
+def data_source_search(filename, query=''):
     """
-    Perform an EQL search on the provided YAML file
+    Perform an EQL search on a data source administration file
     :param filename: file location of the YAML file on disk
-    :param file_type: data source administration file, ...
     :param query: EQL query
-    :param include_all_score_objs: include all score objects within the score_logbook for the EQL query
     :return: a filtered YAML 'file' (i.e. dict) or None when the query was not successful
     """
 
-    if file_type == FILE_TYPE_DATA_SOURCE_ADMINISTRATION:
+    yaml_content_eql, yaml_content_org = _prepare_yaml_file(filename, 'data_sources',
-        obj_type = 'data_sources'
+                                                            include_all_score_objs=False)
-    else:
-        return filename
-
-    yaml_content_eql, yaml_content_org = _prepare_yaml_file(filename, obj_type,
-                                                            include_all_score_objs=include_all_score_objs)
     query_results = _execute_eql_query(yaml_content_eql, query)
 
-    if not _check_query_results(query_results, obj_type):
+    if not _check_query_results(query_results, 'data_sources'):
-        return  # the EQL query was not compatible with the schema
+        return None  # the EQL query was not compatible with the schema
 
-    query_results_yaml = _events_to_yaml(query_results, obj_type)
+    query_results_yaml = _events_to_yaml(query_results, 'data_sources')
 
     if query_results_yaml:
         yaml_content = yaml_content_org
-        yaml_content[obj_type] = query_results_yaml
+        yaml_content['data_sources'] = query_results_yaml
 
         return yaml_content
     else:

generic.py

@@ -581,6 +581,20 @@ def normalize_name_to_filename(name):
     return name.lower().replace(' ', '-')
 
 
+def platform_to_filename(platform):
+    """
+    Makes a filename friendly version of the platform parameter which can be a string or list.
+    :param platform: the platform variable (a string or a list)
+    :return: a filename friendly representation of the value of platform
+    """
+    if platform == 'all':
+        return 'all'
+    elif isinstance(platform, list):
+        return "-".join(platform)
+    else:
+        return ''
+
+
 def map_techniques_to_data_sources(techniques, my_data_sources):
     """
     This function maps the MITRE ATT&CK techniques to your data sources.
@@ -907,6 +921,11 @@ def get_statistics_data_sources():
 
 
 def get_platform_from_yaml(yaml_content):
+    """
+    Read the platform field from the YAML file supporting both string and list values.
+    :param yaml_content: the content of the YAML file containing the platform field
+    :return: the platform value
+    """
     platform = yaml_content['platform']
     if isinstance(platform, str):
         platform = [platform]

health.py

@@ -83,7 +83,7 @@ def _update_health_state_cache(filename, has_error):
                 _update(has_error)
 
 
-def check_health_data_sources(filename, ds_content, health_is_called, no_print=False):
+def check_health_data_sources(filename, ds_content, health_is_called, no_print=False, skip_platform=False):
     """
     Check on errors in the provided data sources administration YAML file.
     :param filename: YAML file location
@@ -96,17 +96,18 @@ def check_health_data_sources(filename, ds_content, health_is_called, no_print=F
 
     platform = ds_content.get('platform', None)
 
-    if platform != 'all' and platform != ['all']:
+    if not skip_platform:
-        if isinstance(platform, str):
+        if platform != 'all' and platform != ['all']:
-            platform = [platform]
+            if isinstance(platform, str):
-        if platform is None or len(platform) == 0 or platform == '':
+                platform = [platform]
-            platform = ['empty']
+            if platform is None or len(platform) == 0 or platform == '':
-        for p in platform:
+                platform = ['empty']
-            if p.lower() not in PLATFORMS.keys():
+            for p in platform:
-                has_error = _print_error_msg(
+                if p.lower() not in PLATFORMS.keys():
-                    '[!] EMPTY or INVALID value for \'platform\' within the data source admin. '
+                    has_error = _print_error_msg(
-                    'file: %s (should be value(s) of: [%s] or all)' % (p, ', '.join(list(PLATFORMS.values()))),
+                        '[!] EMPTY or INVALID value for \'platform\' within the data source admin. '
-                    health_is_called)
+                        'file: %s (should be value(s) of: [%s] or all)' % (p, ', '.join(list(PLATFORMS.values()))),
+                        health_is_called)
 
     for ds in ds_content['data_sources']:
         # check for missing keys

interactive_menu.py

@@ -262,7 +262,7 @@ def _menu_data_source(filename_ds):
         file_ds = filename_ds
 
         if eql_query_data_sources:
-            file_ds = search(filename_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION, eql_query_data_sources)
+            file_ds = data_source_search(filename_ds, eql_query_data_sources)
             if not file_ds:
                 _wait()  # something went wrong in executing the search or 0 results where returned
                 _menu_data_source(filename_ds)

threat-actor-data/20190902-ATTACK-Techniques-and-trends-in-Windows-malware.yaml

@@ -0,0 +1,85 @@
+%YAML 1.2
+---
+version: 1.0
+file_type: group-administration
+groups:
+  - group_name: ATT&CK Techniques and Trends in Windows Malware
+    # Publication: ATT&CK Techniques and Trends in Windows Malware
+    # Authors:     Kris Oosthoek and Christian Doerr
+    # Source:      https://krisk.io/post/sok-attack-securecomm19.pdf
+    campaign:
+    technique_id:
+      T1012: 950
+      T1063: 748
+      T1057: 684
+      T1082: 669
+      T1083: 658
+      T1027: 604
+      T1055: 597
+      T1022: 576
+      T1106: 562
+      T1045: 558
+      T1124: 506
+      T1105: 423
+      T1140: 378
+      T1071: 338
+      T1060: 287
+      T1050: 273
+      T1010: 216
+      T1033: 210
+      T1134: 197
+      T1085: 175
+      T1036: 165
+      T1059: 161
+      T1107: 135
+      T1043: 129
+      T1035: 115
+      T1073: 106
+      T1032: 104
+      T1089: 98
+      T1016: 97
+      T1115: 94
+      T1056: 94
+      T1047: 82
+      T1064: 71
+      T1065: 67
+      T1003: 56
+      T1031: 53
+      T1112: 50
+      T1113: 48
+      T1102: 47
+      T1179: 41
+      T1120: 35
+      T1068: 33
+      T1091: 30
+      T1053: 29
+      T1067: 26
+      T1018: 21
+      T1114: 20
+      T1007: 19
+      T1158: 16
+      T1049: 15
+      T1005: 14
+      T1081: 12
+      T1087: 12
+      T1135: 12
+      T1176: 10
+      T1188: 10
+      T1044: 8
+      T1014: 8
+      T1070: 8
+      T1074: 8
+      T1076: 8
+      T1096: 7
+      T1088: 1 # value < 7
+      T1136: 1 # value < 7
+      T1214: 1 # value < 7
+      T1048: 1 # value < 7
+      T1203: 1 # value < 7
+      T1183: 1 # value < 7
+      T1130: 1 # value < 7
+      T1040: 1 # value < 7
+      T1086: 1 # value < 7
+      T1192: 1 # value < 7
+    software_id: []
+    enabled: True