Merge branch 'development' of https://github.com/rabobank-cdc/DeTTACT into development

# Conflicts:
#	technique_mapping.py

dettact.py

@@ -48,10 +48,11 @@ def init_menu():
     parser_visibility.add_argument('-ft', '--file-tech', help='path to the technique administration YAML file (used to '
                                                               'score the level of visibility)', required=True)
     parser_visibility.add_argument('-fd', '--file-ds', help='path to the data source administration YAML file (used to '
-                                                            'add metadata on the involved data sources)',
+                                                            'add metadata on the involved data sources)')
-                                   required=True)
     parser_visibility.add_argument('-l', '--layer', help='generate a visibility layer for the ATT&CK navigator',
                                    action='store_true')
+    parser_visibility.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques',
+                                     action='store_true')
     parser_visibility.add_argument('-o', '--overlay', help='generate a visibility layer overlayed with detections for '
                                                            'the ATT&CK navigator.', action='store_true')
 
@@ -68,6 +69,8 @@ def init_menu():
                                                            'involved data sources)')
     parser_detection.add_argument('-l', '--layer', help='generate  detection layer for the ATT&CK navigator',
                                   action='store_true')
+    parser_detection.add_argument('-e', '--excel', help='generate an Excel sheet with all administrated techniques',
+                                   action='store_true')
     parser_detection.add_argument('-o', '--overlay', help='generate a detection layer overlayed with visibility for '
                                                           'the ATT&CK navigator.', action='store_true')
     parser_detection.add_argument('-g', '--graph', help='generate a graph with detections added through time',
@@ -142,12 +145,21 @@ def menu(menu_parser):
                 generate_technique_administration_file(args.file)
 
     elif args.subparser in ['visibility', 'v']:
-        if check_file_type(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION) and \
+        if args.layer or args.overlay:
-           check_file_type(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION):
+            if not args.file_ds:
-            if args.layer:
+                print('[!] Generating a visibility layer or doing an overlay requires adding the data source'
-                generate_visibility_layer(args.file_tech, args.file_ds, False)
+                      'administration YAML file (\'--file-ds\')')
-            if args.overlay:
+                quit()
-                generate_visibility_layer(args.file_tech, args.file_ds, True)
+
+            if check_file_type(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION) and \
+               check_file_type(args.file_ds, FILE_TYPE_DATA_SOURCE_ADMINISTRATION):
+                if args.layer:
+                    generate_visibility_layer(args.file_tech, args.file_ds, False)
+                if args.overlay:
+                    generate_visibility_layer(args.file_tech, args.file_ds, True)
+
+        if args.excel and check_file_type(args.file_tech, FILE_TYPE_TECHNIQUE_ADMINISTRATION):
+            export_techniques_list_to_excel(args.file_tech)
 
     elif args.subparser in ['group', 'g']:
         generate_group_heat_map(args.groups, args.overlay, args.overlay_type, args.stage, args.platform, args.software_group)
@@ -167,6 +179,8 @@ def menu(menu_parser):
                 generate_detection_layer(args.file_tech, args.file_ds, True)
             if args.graph:
                 plot_detection_graph(args.file_tech)
+            if args.excel:
+                export_techniques_list_to_excel(args.file_tech)
 
     elif args.subparser in ['generic', 'ge']:
         if args.statistics:

generic.py

@@ -192,7 +192,6 @@ def get_layer_template_detections(name, description, stage, platform):
     :return: layer template dictionary
     """
     layer = _get_base_template(name, description, stage, platform, 0)
-    layer['gradient'] = {'colors': ['#ff6666', '#ffe766', '#8ec843'], 'minValue': 0, 'maxValue': 100}
     layer['legendItems'] = \
         [
             {'label': 'Detection score 0: Forensics/Context', 'color': COLOR_D_0},
@@ -217,7 +216,6 @@ def get_layer_template_data_sources(name, description, stage, platform):
     :return: layer template dictionary
     """
     layer = _get_base_template(name, description, stage, platform, 0)
-    layer['gradient'] = {'colors': ['#ff6666', '#ffe766', '#8ec843'], 'minValue': 0, 'maxValue': 100}
     layer['legendItems'] = \
         [
             {'label': '1-25% of data sources available', 'color': COLOR_DS_25p},
@@ -241,7 +239,6 @@ def get_layer_template_visibility(name, description, stage, platform):
     :return: layer template dictionary
     """
     layer = _get_base_template(name, description, stage, platform, 0)
-    layer['gradient'] = {'colors': ['#ff6666', '#ffe766', '#8ec843'], 'minValue': 0, 'maxValue': 100}
     layer['legendItems'] = \
         [
             {'label': 'Visibility score 1: Minimal', 'color': COLOR_V_1},
@@ -264,7 +261,6 @@ def get_layer_template_layered(name, description, stage, platform):
     :return: layer template dictionary
     """
     layer = _get_base_template(name, description, stage, platform, 0)
-    layer['gradient'] = {'colors': ['#ff6666', '#ffe766', '#8ec843'], 'minValue': 0, 'maxValue': 100}
     layer['legendItems'] = \
         [
             {'label': 'Visibility', 'color': COLOR_OVERLAY_VISIBILITY},
@@ -348,10 +344,13 @@ def check_file_type(filename, file_type=None):
     with open(filename, 'r') as yaml_file:
         try:
             yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader)
-        except:
+        except Exception as e:
             print('[!] File: \'' + filename + '\' is not a valid YAML file.')
+            print('  ' + str(e))  # print more detailed error information to help the user in fixing the error.
             return None
 
+        # This check is performed because a text file will also be considered to be valid YAML. But, we are using
+        # key-value pairs within the YAML files.
         if not hasattr(yaml_content, 'keys'):
             print('[!] File: \'' + filename + '\' is not a valid YAML file.')
             return None
@@ -367,3 +366,20 @@ def check_file_type(filename, file_type=None):
                 return yaml_content['file_type']
         else:
             return yaml_content['file_type']
+
+
+
+def upgrade_technique_yaml_10_to_11(filename):
+    # Load the file:
+    with open(filename, 'r') as yaml_file:
+        yaml_content = yaml.load(yaml_file, Loader=yaml.FullLoader)
+
+    for t in yaml_content['techniques']:
+        t['technique']
+
+    # Save the file:
+    yaml_string = '%YAML 1.2\n---\n' + yaml.dump(yaml_content, sort_keys=False).replace('null', '')
+    output_filename = filename.replace('.yaml', '_copy.yaml')
+    with open(output_filename, 'w') as f:
+        f.write(yaml_string)
+

interactive_menu.py

@@ -259,6 +259,7 @@ def menu_detection(filename_t):
     print('1. Generate a layer for detection coverage for the ATT&CK Navigator.')
     print('2. Generate a layer for detection coverage overlayed with visibility for the ATT&CK Navigator.')
     print('3. Generate a graph with detections added through time.')
+    print('4. Generate an Excel sheet with all administrated techniques.')
     print('9. Back to main menu.')
     choice = ask_input()
     if choice == '1':
@@ -274,6 +275,10 @@ def menu_detection(filename_t):
         print('Drawing the graph...')
         plot_detection_graph(filename_t)
         wait()
+    elif choice == '4':
+        print('Generating Excel file...')
+        export_techniques_list_to_excel(filename_t)
+        wait()
     elif choice == '9':
         interactive_menu()
     elif choice == 'q':
@@ -297,6 +302,7 @@ def menu_visibility(filename_t, filename_ds):
     print('Select what you want to do:')
     print('1. Generate a layer for visibility for the ATT&CK Navigator.')
     print('2. Generate a layers for visibility overlayed with detection coverage for the ATT&CK Navigator.')
+    print('3. Generate an Excel sheet with all administrated techniques.')
     print('9. Back to main menu.')
     choice = ask_input()
     if choice == '1':
@@ -307,6 +313,10 @@ def menu_visibility(filename_t, filename_ds):
         print('Writing visibility coverage layers overlayed with detections...')
         generate_visibility_layer(filename_t, filename_ds, True)
         wait()
+    elif choice == '3':
+        print('Generating Excel file...')
+        export_techniques_list_to_excel(filename_t)
+        wait()
     elif choice == '9':
         interactive_menu()
     elif choice == 'q':

technique_mapping.py

@@ -1,5 +1,6 @@
 import simplejson
 from generic import *
+import xlsxwriter
 # Imports for pandas and plotly are because of performance reasons in the function that uses these libraries.
 
 
@@ -138,27 +139,29 @@ def _map_and_colorize_techniques_for_detections(my_techniques):
     try:
         for d, c in my_techniques.items():
             s = -1 if 'detection' not in c.keys() else c['detection']['score']
-            if 'detection' in c.keys():
+            if s != -1:
-                comment = str(c['detection']['comment']) if str(c['detection']['comment']) != '' else '-'
+                if 'detection' in c.keys():
-            else:
+                    comment = str(c['detection']['comment']) if str(c['detection']['comment']) != '' else '-'
-                comment = '-'
+                else:
-            color = COLOR_D_0 if s == 0 else COLOR_D_1 if s == 1 else COLOR_D_2 if s == 2 else COLOR_D_3 \
+                    comment = '-'
-                if s == 3 else COLOR_D_4 if s == 4 else COLOR_D_5 if s == 5 else ''
+                color = COLOR_D_0 if s == 0 else COLOR_D_1 if s == 1 else COLOR_D_2 if s == 2 else COLOR_D_3 \
-            technique = get_technique(techniques, d)
+                    if s == 3 else COLOR_D_4 if s == 4 else COLOR_D_5 if s == 5 else ''
-            for tactic in technique['tactic']:
+                technique = get_technique(techniques, d)
-                location = ', '.join(c['detection']['location']) if 'detection' in c.keys() else '-'
+                for tactic in technique['tactic']:
-                location = location if location != '' else '-'
+                    location = ', '.join(c['detection']['location']) if 'detection' in c.keys() else '-'
-                applicable_to = ', '.join(c['detection']['applicable_to']) if 'detection' in c.keys() else '-'
+                    location = location if location != '' else '-'
-                x = {}
+                    applicable_to = ', '.join(c['detection']['applicable_to']) if 'detection' in c.keys() else '-'
-                x['techniqueID'] = d
+                    x = {}
-                x['color'] = color
+                    x['techniqueID'] = d
-                x['comment'] = ''
+                    x['color'] = color
-                x['enabled'] = True
+                    x['comment'] = ''
-                x['tactic'] = tactic.lower().replace(' ', '-')
+                    x['enabled'] = True
-                x['metadata'] = [{'name': '-Detection score', 'value': str(s)},
+                    x['tactic'] = tactic.lower().replace(' ', '-')
-                                 {'name': '-Detection location', 'value': location},
+                    x['metadata'] = [{'name': '-Detection score', 'value': str(s)},
-                                 {'name': '-Comment', 'value': comment},
+                                     {'name': '-Detection location', 'value': location},
-                                 {'name': '-Applicable to', 'value': applicable_to}]
+                                     {'name': '-Comment', 'value': comment},
+                                     {'name': '-Applicable to', 'value': applicable_to}]
+                    x['score'] = s
 
                 mapped_techniques.append(x)
     except Exception:
@@ -278,3 +281,79 @@ def _map_and_colorize_techniques_for_overlayed(my_techniques, my_data_sources):
             mapped_techniques.append(x)
 
     return mapped_techniques
+
+
+def export_techniques_list_to_excel(filename):
+    """
+    Makes an overview of the MITRE ATT&CK techniques from the YAML administration file.
+    :param filename: the filename of the yaml file containing the techniques administration
+    :return:
+    """
+    my_techniques, name, platform = _load_detections(filename)
+    my_techniques = dict(sorted(my_techniques.items(), key=lambda kv: kv[0], reverse=False))
+    mitre_techniques = load_attack_data(DATATYPE_ALL_TECH)
+
+    excel_filename = 'output/techniques.xlsx'
+    workbook = xlsxwriter.Workbook(excel_filename)
+    worksheet = workbook.add_worksheet('Data sources')
+
+    # Formatting:
+    format_bold_left = workbook.add_format({'align': 'left', 'bold': True})
+    format_title = workbook.add_format({'align': 'left', 'bold': True, 'font_size': '14'})
+    format_left = workbook.add_format({'align': 'left'})
+    format_bold_center_bggrey = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': '#dbdbdb'})
+    format_bold_center_bgreen = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': '#8bc34a'})
+    format_bold_center_bgblue = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': '#64b5f6'})
+
+    # Title
+    worksheet.write(0, 0, 'Overview of techniques for ' + name, format_title)
+
+    # Header columns
+    worksheet.merge_range(2, 0, 2, 2, 'Technique', format_bold_center_bggrey)
+    worksheet.merge_range(2, 3, 2, 7, 'Detection', format_bold_center_bgreen)
+    worksheet.merge_range(2, 8, 2, 9, 'Visibility', format_bold_center_bgblue)
+    y = 3
+    worksheet.write(y, 0, 'ID', format_bold_left)
+    worksheet.write(y, 1, 'Tactic', format_bold_left)
+    worksheet.write(y, 2, 'Description', format_bold_left)
+    worksheet.write(y, 3, 'Date registered', format_bold_left)
+    worksheet.write(y, 4, 'Date implemented', format_bold_left)
+    worksheet.write(y, 5, 'Score', format_bold_left)
+    worksheet.write(y, 6, 'Location', format_bold_left)
+    worksheet.write(y, 7, 'Comment', format_bold_left)
+    worksheet.write(y, 8, 'Score', format_bold_left)
+    worksheet.write(y, 9, 'Comment', format_bold_left)
+
+    worksheet.set_column(0, 0, 14)
+    worksheet.set_column(1, 1, 50)
+    worksheet.set_column(2, 2, 40)
+    worksheet.set_column(3, 3, 15)
+    worksheet.set_column(4, 4, 18)
+    worksheet.set_column(5, 5, 8)
+    worksheet.set_column(6, 6, 25)
+    worksheet.set_column(7, 7, 40)
+    worksheet.set_column(8, 8, 8)
+    worksheet.set_column(9, 9, 40)
+
+    # Putting the techniques:
+    y = 4
+    for d, c in my_techniques.items():
+        worksheet.write(y, 0, d)
+        worksheet.write(y, 1, ', '.join(t.capitalize() for t in get_technique(mitre_techniques, d)['tactic']))
+        worksheet.write(y, 2, get_technique(mitre_techniques, d)['technique'])
+        worksheet.write(y, 3, str(c['detection']['date_registered']).replace('None', ''))
+        worksheet.write(y, 4, str(c['detection']['date_implemented']).replace('None', ''))
+        worksheet.write(y, 5, c['detection']['score'], format_left)
+        worksheet.write(y, 6, ','.join(c['detection']['location']))
+        worksheet.write(y, 7, c['detection']['comment'])
+        worksheet.write(y, 8, c['visibility']['score'], format_left)
+        worksheet.write(y, 9, c['visibility']['comment'])
+        y += 1
+
+    worksheet.autofilter(3, 0, 3, 9)
+    worksheet.freeze_panes(4, 0)
+    try:
+        workbook.close()
+        print("File written: " + excel_filename)
+    except Exception as e:
+        print('[!] Error while writing Excel file: %s' % str(e))