From 6d172083872b1bcab8423a32e1236354646e783d Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 19 Sep 2019 15:48:38 +0200 Subject: [PATCH 1/5] Fixed the metadata for the data source "Process command-line parameters" --- sample-data/data-sources-endpoints.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample-data/data-sources-endpoints.yaml b/sample-data/data-sources-endpoints.yaml index b20e037..e7d497b 100644 --- a/sample-data/data-sources-endpoints.yaml +++ b/sample-data/data-sources-endpoints.yaml @@ -32,10 +32,10 @@ data_sources: consistency: 0 retention: 0 - data_source_name: Process command-line parameters - date_registered: 2019-03-01 - date_connected: 2017-01-01 - products: [Windows event log] - available_for_data_analytics: True + date_registered: + date_connected: + products: [None] + available_for_data_analytics: False comment: '' data_quality: device_completeness: 0 From 3c341ce9a98ab4bd9f2e32407840b01af798f1b8 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 19 Sep 2019 15:49:59 +0200 Subject: [PATCH 2/5] Fixed a bug within the YAML visibility update functionality. --- constants.py | 4 ++-- data_source_mapping.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/constants.py b/constants.py index 749d15e..bb6e63c 100644 --- a/constants.py +++ b/constants.py @@ -88,9 +88,9 @@ FILE_TYPE_TECHNIQUE_ADMINISTRATION_UPGRADE_TEXT = { " The primary purpose of doing this is to allow you to keep track of changes in the score."} # visibility update questions and answers -V_UPDATE_Q_ALL_MANUAL = 'For all most recent visibility score objects that are eligible for an update. The key-value pair \'auto-generated\' is set to \'false\' or is not present.\n' \ +V_UPDATE_Q_ALL_MANUAL = 'For all most recent visibility score objects that are eligible for an update, the key-value pair \'auto-generated\' is set to \'false\' or is not present.\n' \ 'This implies that these scores are manually assigned. How do you want to proceed?:' -V_UPDATE_Q_ALL_AUTO = 'For all most recent visibility score objects that are eligible for an update. The key-value pair \'auto-generated\' is set to \'true\'. \n' \ +V_UPDATE_Q_ALL_AUTO = 'For all most recent visibility score objects that are eligible for an update, the key-value pair \'auto-generated\' is set to \'true\'. \n' \ 'This implies that these scores are auto-generated. How do you want to proceed?:' V_UPDATE_Q_MIXED = 'You have visibility scores that are eligible for an update, which are manually assigned and which are calculated based on the nr. of data sources (i.e. auto-generated = true)\n' \ 'How do you want to proceed?' diff --git a/data_source_mapping.py b/data_source_mapping.py index 4b0357b..5784f04 100644 --- a/data_source_mapping.py +++ b/data_source_mapping.py @@ -355,9 +355,6 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): if manually_scored and auto_scored: mix_scores = True - manually_scored = False - auto_scored = False - break # stop if none of the present visibility scores are eligible for an update if not mix_scores and not manually_scored and not auto_scored: @@ -366,12 +363,12 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): print('\nA total of ' + str(updated_vis_score_cnt) + ' visibility scores are eligible for an update.\n') # ask how the score should be updated answer = 0 - if manually_scored: + if mix_scores: + answer = ask_multiple_choice(V_UPDATE_Q_MIXED, [V_UPDATE_ANSWER_3, V_UPDATE_ANSWER_4, V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) + elif manually_scored: answer = ask_multiple_choice(V_UPDATE_Q_ALL_MANUAL, [V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) elif auto_scored: answer = ask_multiple_choice(V_UPDATE_Q_ALL_AUTO, [V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) - elif mix_scores: - answer = ask_multiple_choice(V_UPDATE_Q_MIXED, [V_UPDATE_ANSWER_3, V_UPDATE_ANSWER_4, V_UPDATE_ANSWER_1, V_UPDATE_ANSWER_2, V_UPDATE_ANSWER_CANCEL]) if answer == V_UPDATE_ANSWER_CANCEL: return @@ -443,6 +440,7 @@ def update_technique_administration_file(file_data_sources, file_tech_admin): print(' - Date: ' + get_latest_date(old_vis_obj[obj_idx]).strftime('%Y-%m-%d')) print(' - Score: ' + str(get_latest_score(old_vis_obj[obj_idx]))) print(' - Visibility comment: ' + _indent_comment(get_latest_comment(old_vis_obj[obj_idx]), 23)) + print(' - Auto generated: ' + str(get_latest_score_obj(old_vis_obj[obj_idx]).get('auto_generated', 'False'))) print('NEW score object:') print(' - Date: ' + new_score_obj['date']) print(' - Score: ' + str(new_score_obj['score'])) From c287673169952ffb2b58bb827d95a0d151985826 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 19 Sep 2019 15:52:43 +0200 Subject: [PATCH 3/5] Improved the way how EQL is integrated into DeTT&CT. --- eql_yaml.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/eql_yaml.py b/eql_yaml.py index 5629c06..4a7638e 100644 --- a/eql_yaml.py +++ b/eql_yaml.py @@ -86,7 +86,7 @@ def _techniques_to_events(techniques, obj_type, include_all_score_objs): if obj_type == 'detection': # noinspection PyUnboundLocalVariable event_lvl_2['location'] = location - event_lvl_1 = {'event_type': 'techniques', 'technique_id': tech_id, 'technique_name': tech_name, + event_lvl_1 = {'technique_id': tech_id, 'technique_name': tech_name, obj_type: event_lvl_2} technique_events.append(event_lvl_1) @@ -161,9 +161,7 @@ def _events_to_yaml(query_results, obj_type): if obj_type == 'data_sources': try: - # Remove the event_type key. We no longer need this. for r in query_results: - del r['event_type'] if r['date_registered'] and isinstance(r['date_registered'], str): r['date_registered'] = datetime.datetime.strptime(r['date_registered'], '%Y-%m-%d') if r['date_connected'] and isinstance(r['date_connected'], str): @@ -304,18 +302,20 @@ def _prepare_yaml_file(filename, obj_type, include_all_score_objs): yaml_content = _yaml.load(yaml_file) yaml_content_eql = _traverse_modify_date(yaml_content) + yaml_eql_events = [] - # add the event type for EQL + # create EQL events from the list of dictionaries if obj_type == 'data_sources': for item in yaml_content_eql[obj_type]: - item['event_type'] = obj_type - yaml_content_eql = yaml_content_eql['data_sources'] + yaml_eql_events.append(eql.Event(obj_type, 0, item)) - # flatten the technique administration file to events + # flatten the technique administration file to EQL events elif obj_type in ['visibility', 'detection']: yaml_content_eql = _techniques_to_events(yaml_content_eql, obj_type, include_all_score_objs) + for e in yaml_content_eql: + yaml_eql_events.append(eql.Event('techniques', 0, e)) - return yaml_content_eql, yaml_content + return yaml_eql_events, yaml_content def _check_query_results(query_results, obj_type): @@ -353,7 +353,6 @@ def _execute_eql_query(events, query): """ # learn and load the schema schema = eql.Schema.learn(events) - schema.default(schema) query_results = [] @@ -363,14 +362,14 @@ def _execute_eql_query(events, query): # create the engine and parse the query engine = eql.PythonEngine() - with engine.schema: + with schema: try: eql_query = eql.parse_query(query, implied_any=True, implied_base=True) engine.add_query(eql_query) except eql.EqlError as e: print(e, file=sys.stderr) print('\nTake into account the following schema:') - pprint(eql.Schema.current().schema) + pprint(schema.schema) # when using an EQL query that does not match the schema, return None. return None engine.add_output_hook(callback) From 951ab11619605427beb6d55993541be627e421fe Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 19 Sep 2019 16:04:26 +0200 Subject: [PATCH 4/5] Bumped the version to 1.2.1 --- README.md | 2 +- constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f264fbc..ad582a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ DeTT&CT #### Detect Tactics, Techniques & Combat Threats -Latest version: [1.2.0](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-120) +Latest version: [1.2.1](https://github.com/rabobank-cdc/DeTTECT/wiki/Changelog#version-121) To get started with DeTT&CT, check out this [page](https://github.com/rabobank-cdc/DeTTECT/wiki/Getting-started) and our [blog](https://split.to/FkqwE7U). diff --git a/constants.py b/constants.py index bb6e63c..ac6fff5 100644 --- a/constants.py +++ b/constants.py @@ -2,7 +2,7 @@ import re APP_NAME = 'DeTT&CT' APP_DESC = 'Detect Tactics, Techniques & Combat Threats' -VERSION = '1.2.0' +VERSION = '1.2.1' EXPIRE_TIME = 60 * 60 * 24 From 785d52a6194a9fbb1ba2c94e80da8287b7d0b1c2 Mon Sep 17 00:00:00 2001 From: Marcus Bakker Date: Thu, 19 Sep 2019 16:06:51 +0200 Subject: [PATCH 5/5] Bumped the version to 1.2.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c04bd59..2a1edfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.7-alpine -LABEL version="1.1.2" +LABEL version="1.2.1" # update repository and install Linux packages RUN apk update && \