From ed2c886359669ee090c5e795668510c9f8a19f14 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:24:38 +0000 Subject: [PATCH] Magic crib now only applies after all branches have been explored. --- src/core/lib/Magic.mjs | 33 +++++++++++++++------------------ src/core/operations/Magic.mjs | 12 +++++++++--- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 640979c..00132d5 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,10 +265,10 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly - * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output + * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -277,20 +277,17 @@ class Magic { let results = []; // Record the properties of the current data - // Only if there either wasn't a filter provided, - // or the filter matches in the data - if (filter == null || new RegExp(filter).test(this.inputStr)){ - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - } + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful, + matchesCrib: crib && crib.test(this.inputStr) + }); const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -309,7 +306,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); results = results.concat(speculativeResults); })); @@ -321,7 +318,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); + depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib); results = results.concat(bfResults); })); diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 8bfdb06..1555eba 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)."; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; @@ -61,10 +61,16 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang, filter] = ings, + [depth, intensive, extLang, crib] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); + cribRegex = (crib && crib.length) ? new RegExp(crib, "i") : null; + let options = await magic.speculativeExecution(depth, extLang, intensive, [], false, cribRegex); + + // Filter down to results which matched the crib + if (cribRegex) { + options = options.filter(option => option.matchesCrib); + } // Record the current state for use when presenting this.state = state;