557 lines
23 KiB
Python
Executable File
557 lines
23 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import json
|
|
import errno
|
|
import os
|
|
import time
|
|
import traceback
|
|
from nxc.protocols.smb.remotefile import RemoteFile
|
|
from impacket.smb3structs import FILE_READ_DATA
|
|
from impacket.smbconnection import SessionError
|
|
|
|
|
|
CHUNK_SIZE = 4096
|
|
|
|
|
|
def human_size(nbytes):
|
|
"""
|
|
This function takes a number of bytes as input and converts it to a human-readable
|
|
size representation with appropriate units (e.g., KB, MB, GB, TB).
|
|
"""
|
|
suffixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
|
|
|
|
# Find the appropriate unit suffix and convert bytes to higher units
|
|
for i in range(len(suffixes)):
|
|
if nbytes < 1024 or i == len(suffixes) - 1:
|
|
break
|
|
nbytes /= 1024.0
|
|
|
|
# Format the number of bytes with two decimal places and remove trailing zeros and decimal point
|
|
size_str = f"{nbytes:.2f}".rstrip("0").rstrip(".")
|
|
|
|
# Return the human-readable size with the appropriate unit suffix
|
|
return f"{size_str} {suffixes[i]}"
|
|
|
|
|
|
def human_time(timestamp):
|
|
"""This function takes a numerical timestamp (seconds since the epoch) and formats it
|
|
as a human-readable date and time in the format "YYYY-MM-DD HH:MM:SS".
|
|
"""
|
|
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
|
|
|
|
|
|
def make_dirs(path):
|
|
"""
|
|
This function attempts to create directories at the given path. It handles the
|
|
exception `os.errno.EEXIST` that may occur if the directories already exist.
|
|
"""
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
pass
|
|
|
|
|
|
def get_list_from_option(opt):
|
|
"""
|
|
This function takes a comma-separated string and converts it to a list of lowercase strings.
|
|
It filters out empty strings from the input before converting.
|
|
"""
|
|
return list(map(lambda o: o.lower(), filter(bool, opt.split(","))))
|
|
|
|
|
|
class SMBSpiderPlus:
|
|
def __init__(
|
|
self,
|
|
smb,
|
|
logger,
|
|
download_flag,
|
|
stats_flag,
|
|
exclude_exts,
|
|
exclude_filter,
|
|
max_file_size,
|
|
output_folder,
|
|
):
|
|
self.smb = smb
|
|
self.host = self.smb.conn.getRemoteHost()
|
|
self.max_connection_attempts = 5
|
|
self.logger = logger
|
|
self.results = {}
|
|
self.stats = {
|
|
"shares": list(),
|
|
"shares_readable": list(),
|
|
"shares_writable": list(),
|
|
"num_shares_filtered": 0,
|
|
"num_folders": 0,
|
|
"num_folders_filtered": 0,
|
|
"num_files": 0,
|
|
"file_sizes": list(),
|
|
"file_exts": set(),
|
|
"num_get_success": 0,
|
|
"num_get_fail": 0,
|
|
"num_files_filtered": 0,
|
|
"num_files_unmodified": 0,
|
|
"num_files_updated": 0,
|
|
}
|
|
self.download_flag = download_flag
|
|
self.stats_flag = stats_flag
|
|
self.exclude_filter = exclude_filter
|
|
self.exclude_exts = exclude_exts
|
|
self.max_file_size = max_file_size
|
|
self.output_folder = output_folder
|
|
|
|
# Make sure the output_folder exists
|
|
make_dirs(self.output_folder)
|
|
|
|
def reconnect(self):
|
|
"""This function performs a series of reconnection attempts, up to `self.max_connection_attempts`,
|
|
with a 3-second delay between each attempt. It renegotiates the session by creating a new
|
|
connection object and logging in again.
|
|
"""
|
|
for i in range(1, self.max_connection_attempts + 1):
|
|
self.logger.display(f"Reconnection attempt #{i}/{self.max_connection_attempts} to server.")
|
|
|
|
# Renegotiate the session
|
|
time.sleep(3)
|
|
self.smb.create_conn_obj()
|
|
self.smb.login()
|
|
return True
|
|
|
|
return False
|
|
|
|
def list_path(self, share, subfolder):
|
|
"""This function returns a list of paths for a given share/folder."""
|
|
filelist = []
|
|
try:
|
|
# Get file list for the current folder
|
|
filelist = self.smb.conn.listPath(share, subfolder + "*")
|
|
|
|
except SessionError as e:
|
|
self.logger.debug(f'Failed listing files on share "{share}" in folder "{subfolder}".')
|
|
self.logger.debug(str(e))
|
|
|
|
if "STATUS_ACCESS_DENIED" in str(e):
|
|
self.logger.debug(f'Cannot list files in folder "{subfolder}".')
|
|
|
|
elif "STATUS_OBJECT_PATH_NOT_FOUND" in str(e):
|
|
self.logger.debug(f"The folder {subfolder} does not exist.")
|
|
|
|
elif self.reconnect():
|
|
filelist = self.list_path(share, subfolder)
|
|
|
|
return filelist
|
|
|
|
def get_remote_file(self, share, path):
|
|
"""This function will check if a path is readable in a SMB share."""
|
|
try:
|
|
remote_file = RemoteFile(self.smb.conn, path, share, access=FILE_READ_DATA)
|
|
return remote_file
|
|
except SessionError:
|
|
if self.reconnect():
|
|
return self.get_remote_file(share, path)
|
|
|
|
return None
|
|
|
|
def read_chunk(self, remote_file, chunk_size=CHUNK_SIZE):
|
|
"""This function reads the next chunk of data from the provided remote file using
|
|
the specified chunk size. If a `SessionError` is encountered,
|
|
it retries up to 3 times by reconnecting the SMB connection. If the maximum number
|
|
of retries is exhausted or an unexpected exception occurs, it returns an empty chunk.
|
|
"""
|
|
|
|
chunk = ""
|
|
retry = 3
|
|
|
|
while retry > 0:
|
|
retry -= 1
|
|
try:
|
|
chunk = remote_file.read(chunk_size)
|
|
break
|
|
|
|
except SessionError:
|
|
if self.reconnect():
|
|
# Little hack to reset the smb connection instance
|
|
remote_file.__smbConnection = self.smb.conn
|
|
return self.read_chunk(remote_file)
|
|
|
|
except Exception:
|
|
traceback.print_exc()
|
|
break
|
|
|
|
return chunk
|
|
|
|
def get_file_save_path(self, remote_file):
|
|
"""This function processes the remote file path to extract the filename and the folder
|
|
path where the file should be saved locally. It converts forward slashes (/) and backslashes (\)
|
|
in the remote file path to the appropriate path separator for the local file system.
|
|
The folder path and filename are then obtained separately.
|
|
"""
|
|
|
|
# Remove the backslash before the remote host part and replace slashes with the appropriate path separator
|
|
remote_file_path = str(remote_file)[2:].replace("/", os.path.sep).replace("\\", os.path.sep)
|
|
|
|
# Split the path to obtain the folder path and the filename
|
|
folder, filename = os.path.split(remote_file_path)
|
|
|
|
# Join the output folder with the folder path to get the final local folder path
|
|
folder = os.path.join(self.output_folder, folder)
|
|
|
|
return folder, filename
|
|
|
|
def spider_shares(self):
|
|
"""This function enumerates all available shares for the SMB connection, spiders
|
|
through the readable shares, and saves the metadata of the shares to a JSON file.
|
|
"""
|
|
self.logger.info("Enumerating shares for spidering.")
|
|
shares = self.smb.shares()
|
|
|
|
try:
|
|
# Get all available shares for the SMB connection
|
|
for share in shares:
|
|
share_perms = share["access"]
|
|
share_name = share["name"]
|
|
self.stats["shares"].append(share_name)
|
|
|
|
self.logger.info(f'Share "{share_name}" has perms {share_perms}')
|
|
if "WRITE" in share_perms:
|
|
self.stats["shares_writable"].append(share_name)
|
|
if "READ" in share_perms:
|
|
self.stats["shares_readable"].append(share_name)
|
|
else:
|
|
# We only want to spider readable shares
|
|
self.logger.debug(f'Share "{share_name}" not readable.')
|
|
continue
|
|
|
|
# `exclude_filter` is applied to the shares name
|
|
if share_name.lower() in self.exclude_filter:
|
|
self.logger.info(f'Share "{share_name}" has been excluded.')
|
|
self.stats["num_shares_filtered"] += 1
|
|
continue
|
|
|
|
try:
|
|
# Start the spider at the root of the share folder
|
|
self.results[share_name] = {}
|
|
self.spider_folder(share_name, "")
|
|
except SessionError:
|
|
traceback.print_exc()
|
|
self.logger.fail(f"Got a session error while spidering.")
|
|
self.reconnect()
|
|
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
self.logger.fail(f"Error enumerating shares: {str(e)}")
|
|
|
|
# Save the metadata.
|
|
self.dump_folder_metadata(self.results)
|
|
|
|
# Print stats.
|
|
if self.stats_flag:
|
|
self.print_stats()
|
|
|
|
return self.results
|
|
|
|
def spider_folder(self, share_name, folder):
|
|
"""This recursive function traverses through the contents of the specified share and folder.
|
|
It checks each entry (file or folder) against various filters, performs file metadata recording,
|
|
and downloads eligible files if the download flag is set.
|
|
"""
|
|
self.logger.info(f'Spider share "{share_name}" in folder "{folder}".')
|
|
|
|
filelist = self.list_path(share_name, folder + "*")
|
|
|
|
# For each entry:
|
|
# - It's a folder then we spider it (skipping `.` and `..`)
|
|
# - It's a file then we apply the checks
|
|
for result in filelist:
|
|
next_filedir = result.get_longname()
|
|
if next_filedir in [".", ".."]:
|
|
continue
|
|
next_fullpath = folder + next_filedir
|
|
result_type = "folder" if result.is_directory() else "file"
|
|
self.stats[f"num_{result_type}s"] += 1
|
|
|
|
# Check file-dir exclusion filter.
|
|
if any(d in next_filedir.lower() for d in self.exclude_filter):
|
|
self.logger.info(f'The {result_type} "{next_filedir}" has been excluded')
|
|
self.stats[f"{result_type}s_filtered"] += 1
|
|
continue
|
|
|
|
if result_type == "folder":
|
|
self.logger.info(f'Current folder in share "{share_name}": "{next_fullpath}"')
|
|
self.spider_folder(share_name, next_fullpath + "/")
|
|
else:
|
|
self.logger.info(f'Current file in share "{share_name}": "{next_fullpath}"')
|
|
self.parse_file(share_name, next_fullpath, result)
|
|
|
|
def parse_file(self, share_name, file_path, file_info):
|
|
"""This function checks file attributes against various filters, records file metadata,
|
|
and downloads eligible files if the download flag is set.
|
|
"""
|
|
|
|
# Record the file metadata
|
|
file_size = file_info.get_filesize()
|
|
file_creation_time = file_info.get_ctime_epoch()
|
|
file_modified_time = file_info.get_mtime_epoch()
|
|
file_access_time = file_info.get_atime_epoch()
|
|
self.results[share_name][file_path] = {
|
|
"size": human_size(file_size),
|
|
"ctime_epoch": human_time(file_creation_time),
|
|
"mtime_epoch": human_time(file_modified_time),
|
|
"atime_epoch": human_time(file_access_time),
|
|
}
|
|
self.stats["file_sizes"].append(file_size)
|
|
|
|
# Check if proceeding with download attempt.
|
|
if not self.download_flag:
|
|
return
|
|
|
|
# Check file extension filter.
|
|
_, file_extension = os.path.splitext(file_path)
|
|
if file_extension:
|
|
self.stats["file_exts"].add(file_extension.lower())
|
|
if file_extension.lower() in self.exclude_exts:
|
|
self.logger.info(f'The file "{file_path}" has an excluded extension.')
|
|
self.stats["num_files_filtered"] += 1
|
|
return
|
|
|
|
# Check file size limits.
|
|
if file_size > self.max_file_size:
|
|
self.logger.info(f"File {file_path} has size {human_size(file_size)} > max size {human_size(self.max_file_size)}.")
|
|
self.stats["num_files_filtered"] += 1
|
|
return
|
|
|
|
# Check if the remote file is readable.
|
|
remote_file = self.get_remote_file(share_name, file_path)
|
|
if not remote_file:
|
|
self.logger.fail(f'Cannot read remote file "{file_path}".')
|
|
self.stats["num_get_fail"] += 1
|
|
return
|
|
|
|
# Check if the file is already downloaded and up-to-date.
|
|
file_dir, file_name = self.get_file_save_path(remote_file)
|
|
download_path = os.path.join(file_dir, file_name)
|
|
needs_update_flag = False
|
|
if os.path.exists(download_path):
|
|
if file_modified_time <= os.stat(download_path).st_mtime and os.path.getsize(download_path) == file_size:
|
|
self.logger.info(f'File already downloaded "{file_path}" => "{download_path}".')
|
|
self.stats["num_files_unmodified"] += 1
|
|
return
|
|
else:
|
|
needs_update_flag = True
|
|
|
|
# Download file.
|
|
download_success = False
|
|
try:
|
|
self.logger.info(f'Downloading file "{file_path}" => "{download_path}".')
|
|
remote_file.open()
|
|
self.save_file(remote_file, share_name)
|
|
remote_file.close()
|
|
download_success = True
|
|
except SessionError as e:
|
|
if "STATUS_SHARING_VIOLATION" in str(e):
|
|
pass
|
|
except Exception as e:
|
|
self.logger.fail(f'Failed to download file "{file_path}". Error: {str(e)}')
|
|
|
|
# Increment stats counters
|
|
if download_success:
|
|
self.stats["num_get_success"] += 1
|
|
if needs_update_flag:
|
|
self.stats["num_files_updated"] += 1
|
|
else:
|
|
self.stats["num_get_fail"] += 1
|
|
|
|
def save_file(self, remote_file, share_name):
|
|
"""This function reads the `remote_file` in chunks using the `read_chunk` method.
|
|
Each chunk is then written to the local file until the entire file is saved.
|
|
It handles cases where the file remains empty due to errors.
|
|
"""
|
|
|
|
# Reset the remote_file to point to the beginning of the file.
|
|
remote_file.seek(0, 0)
|
|
|
|
folder, filename = self.get_file_save_path(remote_file)
|
|
download_path = os.path.join(folder, filename)
|
|
|
|
# Create the subdirectories based on the share name and file path.
|
|
self.logger.debug(f'Create folder "{folder}"')
|
|
make_dirs(folder)
|
|
|
|
try:
|
|
with open(download_path, "wb") as fd:
|
|
while True:
|
|
chunk = self.read_chunk(remote_file)
|
|
if not chunk:
|
|
break
|
|
fd.write(chunk)
|
|
except Exception as e:
|
|
self.logger.fail(f'Error writing file "{remote_path}" from share "{share_name}": {e}')
|
|
|
|
# Check if the file is empty and should not be.
|
|
if os.path.getsize(download_path) == 0 and remote_file.get_filesize() > 0:
|
|
os.remove(download_path)
|
|
remote_path = str(remote_file)[2:]
|
|
self.logger.fail(f'Unable to download file "{remote_path}".')
|
|
|
|
def dump_folder_metadata(self, results):
|
|
"""This function takes the metadata results as input and writes them to a JSON file
|
|
in the `self.output_folder`. The results are formatted with indentation and
|
|
sorted keys before being written to the file.
|
|
"""
|
|
metadata_path = os.path.join(self.output_folder, f"{self.host}.json")
|
|
try:
|
|
with open(metadata_path, "w", encoding="utf-8") as fd:
|
|
fd.write(json.dumps(results, indent=4, sort_keys=True))
|
|
self.logger.success(f'Saved share-file metadata to "{metadata_path}".')
|
|
except Exception as e:
|
|
self.logger.fail(f"Failed to save share metadata: {str(e)}")
|
|
|
|
def print_stats(self):
|
|
"""This function prints the statistics during processing."""
|
|
|
|
# Share statistics.
|
|
shares = self.stats.get("shares", [])
|
|
if shares:
|
|
num_shares = len(shares)
|
|
shares_str = ", ".join(shares)
|
|
self.logger.display(f"SMB Shares: {num_shares} ({shares_str})")
|
|
shares_readable = self.stats.get("shares_readable", [])
|
|
if shares_readable:
|
|
num_readable_shares = len(shares_readable)
|
|
if len(shares_readable) > 10:
|
|
shares_readable_str = ", ".join(shares_readable[:10]) + "..."
|
|
else:
|
|
shares_readable_str = ", ".join(shares_readable)
|
|
self.logger.display(f"SMB Readable Shares: {num_readable_shares} ({shares_readable_str})")
|
|
shares_writable = self.stats.get("shares_writable", [])
|
|
if shares_writable:
|
|
num_writable_shares = len(shares_writable)
|
|
if len(shares_writable) > 10:
|
|
shares_writable_str = ", ".join(shares_writable[:10]) + "..."
|
|
else:
|
|
shares_writable_str = ", ".join(shares_writable)
|
|
self.logger.display(f"SMB Writable Shares: {num_writable_shares} ({shares_writable_str})")
|
|
num_shares_filtered = self.stats.get("num_shares_filtered", 0)
|
|
if num_shares_filtered:
|
|
self.logger.display(f"SMB Filtered Shares: {num_shares_filtered}")
|
|
|
|
# Folder statistics.
|
|
num_folders = self.stats.get("num_folders", 0)
|
|
self.logger.display(f"Total folders found: {num_folders}")
|
|
num_folders_filtered = self.stats.get("num_folders_filtered", 0)
|
|
if num_folders_filtered:
|
|
num_filtered_folders = len(num_folders_filtered)
|
|
self.logger.display(f"Folders Filtered: {num_filtered_folders}")
|
|
|
|
# File statistics.
|
|
num_files = self.stats.get("num_files", 0)
|
|
self.logger.display(f"Total files found: {num_files}")
|
|
num_files_filtered = self.stats.get("num_files_filtered", 0)
|
|
if num_files_filtered:
|
|
self.logger.display(f"Files filtered: {num_files_filtered}")
|
|
if num_files == 0:
|
|
return
|
|
|
|
# File sizing statistics.
|
|
file_sizes = self.stats.get("file_sizes", [])
|
|
if file_sizes:
|
|
total_file_size = sum(file_sizes)
|
|
min_file_size = min(file_sizes)
|
|
max_file_size = max(file_sizes)
|
|
average_file_size = total_file_size / num_files
|
|
self.logger.display(f"File size average: {human_size(average_file_size)}")
|
|
self.logger.display(f"File size min: {human_size(min_file_size)}")
|
|
self.logger.display(f"File size max: {human_size(max_file_size)}")
|
|
|
|
# Extension statistics.
|
|
file_exts = list(self.stats.get("file_exts", []))
|
|
if file_exts:
|
|
num_unique_file_exts = len(file_exts)
|
|
if len(file_exts) > 10:
|
|
unique_exts_str = ", ".join(file_exts[:10]) + "..."
|
|
else:
|
|
unique_exts_str = ", ".join(file_exts)
|
|
self.logger.display(f"File unique exts: {num_unique_file_exts} ({unique_exts_str})")
|
|
|
|
# Download statistics.
|
|
if self.download_flag:
|
|
num_get_success = self.stats.get("num_get_success", 0)
|
|
if num_get_success:
|
|
self.logger.display(f"Downloads successful: {num_get_success}")
|
|
num_get_fail = self.stats.get("num_get_fail", 0)
|
|
if num_get_fail:
|
|
self.logger.display(f"Downloads failed: {num_get_fail}")
|
|
num_files_unmodified = self.stats.get("num_files_unmodified", 0)
|
|
if num_files_unmodified:
|
|
self.logger.display(f"Unmodified files: {num_files_unmodified}")
|
|
num_files_updated = self.stats.get("num_files_updated", 0)
|
|
if num_files_updated:
|
|
self.logger.display(f"Updated files: {num_files_updated}")
|
|
if num_files_unmodified and not num_files_updated:
|
|
self.logger.display("All files were not changed.")
|
|
if num_files_filtered == num_files:
|
|
self.logger.display("All files were ignored.")
|
|
if num_get_fail == 0:
|
|
self.logger.success("All files processed successfully.")
|
|
|
|
|
|
class NXCModule:
|
|
"""
|
|
Spider plus module
|
|
Module by @vincd
|
|
Updated by @godylockz
|
|
"""
|
|
|
|
name = "spider_plus"
|
|
description = "List files recursively (excluding `EXCLUDE_FILTER` and `EXCLUDE_EXTS` extensions) and save JSON share-file metadata to the `OUTPUT_FOLDER`. If `DOWNLOAD_FLAG`=True, download files smaller then `MAX_FILE_SIZE` to the `OUTPUT_FOLDER`."
|
|
supported_protocols = ["smb"]
|
|
opsec_safe = True # Does the module touch disk?
|
|
multiple_hosts = True # Does the module support multiple hosts?
|
|
|
|
def options(self, context, module_options):
|
|
"""
|
|
DOWNLOAD_FLAG Download all share folders/files (Default: False)
|
|
STATS_FLAG Disable file/download statistics (Default: True)
|
|
EXCLUDE_EXTS Case-insensitive extension filter to exclude (Default: ico,lnk)
|
|
EXCLUDE_FILTER Case-insensitive filter to exclude folders/files (Default: print$,ipc$)
|
|
MAX_FILE_SIZE Max file size to download (Default: 51200)
|
|
OUTPUT_FOLDER Path of the local folder to save files (Default: /tmp/nxc_spider_plus)
|
|
"""
|
|
self.download_flag = False
|
|
if any("DOWNLOAD" in key for key in module_options.keys()):
|
|
self.download_flag = True
|
|
self.stats_flag = True
|
|
if any("STATS" in key for key in module_options.keys()):
|
|
self.stats_flag = False
|
|
self.exclude_exts = get_list_from_option(module_options.get("EXCLUDE_EXTS", "ico,lnk"))
|
|
self.exclude_exts = [d.lower() for d in self.exclude_exts] # force case-insensitive
|
|
self.exclude_filter = get_list_from_option(module_options.get("EXCLUDE_FILTER", "print$,ipc$"))
|
|
self.exclude_filter = [d.lower() for d in self.exclude_filter] # force case-insensitive
|
|
self.max_file_size = int(module_options.get("MAX_FILE_SIZE", 50 * 1024))
|
|
self.output_folder = module_options.get("OUTPUT_FOLDER", os.path.join("/tmp", "nxc_spider_plus"))
|
|
|
|
|
|
def on_login(self, context, connection):
|
|
context.log.display("Started module spidering_plus with the following options:")
|
|
context.log.display(f" DOWNLOAD_FLAG: {self.download_flag}")
|
|
context.log.display(f" STATS_FLAG: {self.stats_flag}")
|
|
context.log.display(f"EXCLUDE_FILTER: {self.exclude_filter}")
|
|
context.log.display(f" EXCLUDE_EXTS: {self.exclude_exts}")
|
|
context.log.display(f" MAX_FILE_SIZE: {human_size(self.max_file_size)}")
|
|
context.log.display(f" OUTPUT_FOLDER: {self.output_folder}")
|
|
|
|
spider = SMBSpiderPlus(
|
|
connection,
|
|
context.log,
|
|
self.download_flag,
|
|
self.stats_flag,
|
|
self.exclude_exts,
|
|
self.exclude_filter,
|
|
self.max_file_size,
|
|
self.output_folder,
|
|
)
|
|
|
|
spider.spider_shares()
|