- Non-MITRE ATT&CK data sources are now also exported to Excel.

- Any ATT&CK data sources that are missing within the YAML file are added to the Excel with a comment stating it is missing.

constants.py

@@ -144,6 +144,19 @@ YAML_OBJ_TECHNIQUE = {'technique_id': '',
                       'detection': YAML_OBJ_DETECTION,
                       'visibility': YAML_OBJ_VISIBILITY}
 
+YAML_OBJ_DATA_SOURCE = {'data_source_name': '',
+                        'date_registered': None,
+                        'date_connected': None,
+                        'products': [''],
+                        'available_for_data_analytics': False,
+                        'comment': '',
+                        'data_quality': {
+                            'device_completeness': 0,
+                            'data_field_completeness': 0,
+                            'timeliness': 0,
+                            'consistency': 0,
+                            'retention': 0}}
+
 # Interactive menu
 MENU_NAME_DATA_SOURCE_MAPPING = 'Data source mapping'
 MENU_NAME_VISIBILITY_MAPPING = 'Visibility coverage mapping'

data_source_mapping.py

@@ -112,10 +112,19 @@ def export_data_source_list_to_excel(filename):
 
     # Putting the data sources data:
     y = 3
-    for d in get_all_mitre_data_sources():
-        worksheet.write(y, 0, d, valign_top)
-        if d in my_data_sources.keys():
+
+    # check if an ATT&CK data source is missing from the data source YAML administration file
+    my_ds_list = my_data_sources.keys()
+    for ds in get_all_mitre_data_sources():
+        if ds not in my_ds_list:
+            ds_obj = deepcopy(YAML_OBJ_DATA_SOURCE)
+            ds_obj['data_source_name'] = ds
+            ds_obj['comment'] = 'ATT&CK data source is missing from the YAML file'
+            my_data_sources[ds] = ds_obj
+
+    for d in sorted(my_data_sources.keys()):
         ds = my_data_sources[d]
+        worksheet.write(y, 0, d, valign_top)
 
         date_registered = ds['date_registered'].strftime('%Y-%m-%d') if isinstance(ds['date_registered'], datetime) else ds['date_registered']
         date_connected = ds['date_connected'].strftime('%Y-%m-%d') if isinstance(ds['date_connected'], datetime) else ds['date_connected']

generic.py

@@ -579,7 +579,7 @@ def map_techniques_to_data_sources(techniques, my_data_sources):
 
 def get_all_mitre_data_sources():
     """
-    Gets all the data sources from the techniques and make a unique sorted list of it.
+    Gets all the data sources from the techniques and make a set.
     :return: a sorted list with all data sources
     """
     techniques = load_attack_data(DATA_TYPE_STIX_ALL_TECH)
@@ -589,7 +589,7 @@ def get_all_mitre_data_sources():
         if 'x_mitre_data_sources' in t.keys():
             for ds in t['x_mitre_data_sources']:
                 data_sources.add(ds)
-    return sorted(data_sources)
+    return data_sources
 
 
 def calculate_score(list_detections, zero_value=0):