From 8a8be0f7833bacd6d05deebac7b086dfbfc4d356 Mon Sep 17 00:00:00 2001 From: Sunny Neo Date: Sat, 8 Aug 2020 01:42:33 +0800 Subject: [PATCH] Update ReadFile to raw.githubusercontent and handle HTTP Response Conflict like TooManyRequest --- Src/Common/FSecure/Github/GithubApi.cpp | 77 +++++++++---------------- Src/Common/FSecure/Github/GithubApi.h | 2 +- 2 files changed, 28 insertions(+), 51 deletions(-) diff --git a/Src/Common/FSecure/Github/GithubApi.cpp b/Src/Common/FSecure/Github/GithubApi.cpp index b25074b..6f84ae6 100644 --- a/Src/Common/FSecure/Github/GithubApi.cpp +++ b/Src/Common/FSecure/Github/GithubApi.cpp @@ -98,40 +98,27 @@ std::string FSecure::GithubApi::CreateChannel(std::string const& channelName) { return channelName; } -FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSizeSHA) { +FSecure::ByteVector FSecure::GithubApi::ReadFile(std::string const& fileNameSHA) { std::string url; json response; - std::string delimiter = OBF(":"); - - //string contains filename:size:sha value - std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); - - std::int64_t fileSize = 0; + std::string delimiter = OBF("!"); std::string filename; std::string fileSHA; + std::string fileDownloadURL; + //string contains filename:sha:download_url value + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); - if (fileNameSizeSHASplit.size() > 0) { - filename = fileNameSizeSHASplit.at(0); - fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL ,10); - fileSHA = fileNameSizeSHASplit.at(2); + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); + fileDownloadURL = fileNameSHASplit.at(2); } else { - throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); } - - // Github does not allow API access to download raw file with size of > 1 MB - // less than or equal to 1 MB for Github API, otherwise Github Blob to retrieve the file - if (fileSize <= 1048576) { - url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/contents/") + filename; - } - else { - url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel - + OBF("/git/blobs/") + fileSHA; - } - - auto content = cppcodec::base64_rfc4648::decode(SendHttpRequest(url, OBF("application/vnd.github.v3.raw"), Method::GET, true)); + + ByteVector content = SendHttpRequest(fileDownloadURL, "", Method::GET, true); return content; } @@ -156,14 +143,12 @@ void FSecure::GithubApi::WriteMessageToFile(std::string const& direction, ByteVi j[OBF("message")] = OBF("Initial Commit"); j[OBF("branch")] = OBF("master"); - j[OBF("content")] = cppcodec::base64_rfc4648::encode(cppcodec::base64_rfc4648::encode(data)); + j[OBF("content")] = cppcodec::base64_rfc4648::encode(data); json response = SendJsonRequest(url, j, Method::PUT); } void FSecure::GithubApi::UploadFile(std::string const& path) { - std::cout << "UPLOAD FILE ... " << std::flush; - std::filesystem::path filepathForUpload = path; auto readFile = std::ifstream(filepathForUpload, std::ios::binary); @@ -175,30 +160,25 @@ void FSecure::GithubApi::UploadFile(std::string const& path) { std::string filename = OBF("upload-") + FSecure::Utils::GenerateRandomString(10) + OBF("-") + ts + OBF("-") + fn; WriteMessageToFile("", packet, filename); - - std::cout << "UPLOAD FILE OK" << std::endl; } -void FSecure::GithubApi::DeleteFile(std::string const& fileNameSizeSHA) { +void FSecure::GithubApi::DeleteFile(std::string const& fileNameSHA) { std::string url; json j; json response; - std::string delimiter = OBF(":"); + std::string delimiter = OBF("!"); std::string fileSHA; - - std::vector fileNameSizeSHASplit = Utils::SplitAndCopy(fileNameSizeSHA, delimiter); - - std::int64_t fileSize; std::string filename; - if (fileNameSizeSHASplit.size() > 0) { - filename = fileNameSizeSHASplit.at(0); - fileSize = std::stoull(fileNameSizeSHASplit.at(1), NULL, 10); - fileSHA = fileNameSizeSHASplit.at(2); + std::vector fileNameSHASplit = Utils::SplitAndCopy(fileNameSHA, delimiter); + + if (fileNameSHASplit.size() > 0) { + filename = fileNameSHASplit.at(0); + fileSHA = fileNameSHASplit.at(1); } else { - throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSizeSHA\n")); + throw std::runtime_error(OBF("Throwing exception: cant parse fileNameSHA\n")); } url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel @@ -230,10 +210,9 @@ std::map FSecure::GithubApi::GetMessagesByDirection(st json response; std::string filename; std::size_t found; - std::int64_t fileSize; - std::string fileSizeStr; std::string fileSHA; - + std::string fileDownloadURL; + std::string delimiter = OBF("!"); std::string url = OBF("https://api.github.com/repos/") + this->m_Username + OBF("/") + this->m_Channel + OBF("/contents"); @@ -242,17 +221,15 @@ std::map FSecure::GithubApi::GetMessagesByDirection(st for (auto& match : response) { if (match.contains(OBF("name"))) { filename = match[OBF("name")]; - fileSize = match[OBF("size")]; fileSHA = match[OBF("sha")]; - - fileSizeStr = std::to_string(fileSize); + fileDownloadURL = match[OBF("download_url")]; //Search whether filename contains direction id found = filename.find(direction); - + if (found != std::string::npos) { std::string ts = filename.substr(filename.length() - 10); // 10 = epoch time length - messages.insert({ts, filename + OBF(":") + fileSizeStr + OBF(":") + fileSHA }); + messages.insert({ts, filename + delimiter + fileSHA + delimiter + fileDownloadURL}); } } } @@ -285,7 +262,7 @@ FSecure::ByteVector FSecure::GithubApi::SendHttpRequest(std::string const& host, if (resp.GetStatusCode() == StatusCode::OK || resp.GetStatusCode() == StatusCode::Created) { return resp.GetData(); } - else if (resp.GetStatusCode() == StatusCode::TooManyRequests) { + else if (resp.GetStatusCode() == StatusCode::TooManyRequests || resp.GetStatusCode() == StatusCode::Conflict) { std::this_thread::sleep_for(Utils::GenerateRandomValue(10s, 20s)); } else { diff --git a/Src/Common/FSecure/Github/GithubApi.h b/Src/Common/FSecure/Github/GithubApi.h index 389f73d..411ba8e 100644 --- a/Src/Common/FSecure/Github/GithubApi.h +++ b/Src/Common/FSecure/Github/GithubApi.h @@ -43,7 +43,7 @@ namespace FSecure /// Download file by its path. /// @param filename - path of file and the size. Format "filename:filesize" /// @return - string of file content - FSecure::ByteVector ReadFile(std::string const& fileNameSizeSHA); + FSecure::ByteVector ReadFile(std::string const& fileNameSHA); /// Write a message as the contents of a file and upload to Github. /// @param direction - the name of the file to upload