diff --git a/dDashboardServer/routes/CMakeLists.txt b/dDashboardServer/routes/CMakeLists.txt index b2d4049f6..6d2b8b8cf 100644 --- a/dDashboardServer/routes/CMakeLists.txt +++ b/dDashboardServer/routes/CMakeLists.txt @@ -58,6 +58,7 @@ set(DASHBOARDROUTES_SOURCES "ContrabandRoutes.cpp" "UgcRoutes.cpp" "UgcLinks.cpp" + "UgcFetch.cpp" "PropertyRentRoutes.cpp" "PasswordRecovery.cpp" "PrometheusMetrics.cpp" diff --git a/dDashboardServer/routes/UgcFetch.cpp b/dDashboardServer/routes/UgcFetch.cpp new file mode 100644 index 000000000..7de5475f9 --- /dev/null +++ b/dDashboardServer/routes/UgcFetch.cpp @@ -0,0 +1,102 @@ +#include "UgcFetch.h" + +#include +#include + +#include + +#include "dConfig.h" +#include "Game.h" +#include "RouteUtils.h" +#include "TtlCache.h" + +namespace { + std::mutex g_CacheMutex; + TtlCache> g_FileCache(std::chrono::seconds(15), 32 * 1024 * 1024); + TtlCache> g_StatusCache(std::chrono::seconds(2), 1024 * 1024); + + constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024; + + size_t Collect(char* data, size_t size, size_t count, void* userData) { + auto* out = static_cast(userData); + if (out->size() + size * count > MAX_FETCH_BYTES) return 0; + out->append(data, size * count); + return size * count; + } + + // Sends a request set up by setup (a GET unless it says otherwise) + template + std::shared_ptr Perform(const std::string& url, long timeout, Setup&& setup) { + auto out = std::make_shared(); + CURL* curl = curl_easy_init(); + if (!curl) { + out->error = "could not start a request"; + return out; + } + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); + setup(curl); + const auto code = curl_easy_perform(curl); + if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); + else out->error = curl_easy_strerror(code); + curl_easy_cleanup(curl); + return out; + } +} + +namespace UgcFetch { + std::string InternalUrl() { + auto url = Game::config->GetValue("ugc_internal_url"); + if (url.empty()) url = "http://127.0.0.1:2008"; + while (url.ends_with('/')) url.pop_back(); + return url; + } + + std::shared_ptr Get(const std::string& url) { + return Perform(url, 20L, [](CURL*) {}); + } + + std::shared_ptr CachedGet(const std::string& url, eCache cache) { + auto& store = cache == eCache::STATUS ? g_StatusCache : g_FileCache; + { + std::lock_guard lock(g_CacheMutex); + if (auto hit = store.Get(url)) return *hit; + } + auto fetched = Get(url); + if (fetched->status == 200 || fetched->status == 404) { + std::lock_guard lock(g_CacheMutex); + store.Put(url, fetched, fetched->body.size() + 64); + } + return fetched; + } + + std::shared_ptr AdminPost(const std::string& url, const std::string& key, const std::string& body) { + curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json"); + headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str()); + auto out = Perform(url, 120L, [&](CURL* curl) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); + }); + curl_slist_free_all(headers); + return out; + } + + void ReplyError(HTTPReply& reply, const Fetched& fetched, std::string_view shownUrl) { + using RouteUtils::JsonError; + if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment"); + if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has not made it"); + if (fetched.status == 0) { + const std::string at = shownUrl.empty() ? std::string() : " at " + std::string(shownUrl); + return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer" + at + " (" + fetched.error + ")"); + } + JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); + } +} diff --git a/dDashboardServer/routes/UgcFetch.h b/dDashboardServer/routes/UgcFetch.h new file mode 100644 index 000000000..54117421c --- /dev/null +++ b/dDashboardServer/routes/UgcFetch.h @@ -0,0 +1,44 @@ +#ifndef UGCFETCH_H +#define UGCFETCH_H + +#include +#include +#include + +struct HTTPReply; + +/** + * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files it + * made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Work out the + * URL (and anything else from the config or database) on the web thread; the requests themselves only touch curl and + * this file's cache, so they can run on worker threads (Workers::Reply). + */ +namespace UgcFetch { + struct Fetched { + long status{}; // HTTP status, 0 when the UGC server didn't answer + std::string body; + std::string error; + }; + + enum class eCache { + FILE, // files the UGC server made, kept 15 seconds + STATUS, // its status, kept 2 seconds + }; + + // Web thread: where the dashboard reaches the UGC server, without a trailing slash + std::string InternalUrl(); + + // Any thread: a GET to the UGC server, not cached + std::shared_ptr Get(const std::string& url); + + // Any thread: Get through a short cache (answers that aren't a file being made are kept) + std::shared_ptr CachedGet(const std::string& url, eCache cache = eCache::FILE); + + // Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks + std::shared_ptr AdminPost(const std::string& url, const std::string& key, const std::string& body); + + // Answers with why a fetch didn't give a file. The URL is only named when shown (not to players, who don't need it) + void ReplyError(HTTPReply& reply, const Fetched& fetched, std::string_view shownUrl = {}); +} + +#endif // !UGCFETCH_H diff --git a/dDashboardServer/routes/UgcLinks.cpp b/dDashboardServer/routes/UgcLinks.cpp index 9cd82b1af..a99aad54c 100644 --- a/dDashboardServer/routes/UgcLinks.cpp +++ b/dDashboardServer/routes/UgcLinks.cpp @@ -2,11 +2,8 @@ #include #include -#include #include -#include - #include "Database.h" #include "dConfig.h" #include "eHTTPMethod.h" @@ -14,83 +11,19 @@ #include "GeneralUtils.h" #include "NifFile.h" #include "RouteUtils.h" -#include "TtlCache.h" +#include "UgcFetch.h" #include "UgcLookup.h" #include "Workers.h" using namespace RouteUtils; using namespace UgcLookup; +using namespace UgcFetch; namespace { constexpr uint32_t SEARCH_LIMIT = 50; // Creators whose inventories a search looks in for creations that are not placed or mailed constexpr size_t MAX_INVENTORIES = 25; - // ---- The UGC server's files, fetched by the dashboard (as the /ugc page's routes do) ---- - - struct Fetched { - long status{}; // HTTP status, 0 when the UGC server didn't answer - std::string body; - std::string error; - }; - - std::mutex g_CacheMutex; - TtlCache> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024); - constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024; - - size_t Collect(char* data, size_t size, size_t count, void* userData) { - auto* out = static_cast(userData); - if (out->size() + size * count > MAX_FETCH_BYTES) return 0; - out->append(data, size * count); - return size * count; - } - - // Web thread - std::string InternalUrl() { - auto url = Game::config->GetValue("ugc_internal_url"); - if (url.empty()) url = "http://127.0.0.1:2008"; - while (url.ends_with('/')) url.pop_back(); - return url; - } - - // Any thread - std::shared_ptr Fetch(const std::string& url) { - { - std::lock_guard lock(g_CacheMutex); - if (auto hit = g_Cache.Get(url)) return *hit; - } - auto out = std::make_shared(); - CURL* curl = curl_easy_init(); - if (!curl) { - out->error = "could not start a request"; - return out; - } - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard"); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); - const auto code = curl_easy_perform(curl); - if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); - else out->error = curl_easy_strerror(code); - curl_easy_cleanup(curl); - if (out->status == 200 || out->status == 404) { - std::lock_guard lock(g_CacheMutex); - g_Cache.Put(url, out, out->body.size() + 64); - } - return out; - } - - void FetchError(HTTPReply& reply, const Fetched& fetched) { - if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment"); - if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has not made it"); - if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer (" + fetched.error + ")"); - JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); - } // ---- Who may see a creation ---- @@ -298,8 +231,8 @@ namespace UgcLinks { if (!MaySee(context, *kind, *id)) return JsonError(reply, eHTTPStatusCode::FORBIDDEN, "You may not view this creation"); const auto url = InternalUrl() + "/files/" + KindName(*kind) + "/" + std::to_string(*id) + "/icon.png"; Workers::Reply(reply, context, false, [url](HTTPReply& out) { - const auto fetched = Fetch(url); - if (fetched->status != 200) return FetchError(out, *fetched); + const auto fetched = CachedGet(url); + if (fetched->status != 200) return ReplyError(out, *fetched); out.status = eHTTPStatusCode::OK; out.contentType = eContentType::IMAGE_PNG; out.message = fetched->body; @@ -317,8 +250,8 @@ namespace UgcLinks { const auto lod = std::min(GeneralUtils::TryParse(QueryValue(context.queryString, "lod")).value_or(0), 3u); const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/model.nif"; Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) { - const auto fetched = Fetch(url); - if (fetched->status != 200) return FetchError(out, *fetched); + const auto fetched = CachedGet(url); + if (fetched->status != 200) return ReplyError(out, *fetched); std::string error; const auto model = NifFile::Parse(fetched->body, lod, error); if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error); diff --git a/dDashboardServer/routes/UgcRoutes.cpp b/dDashboardServer/routes/UgcRoutes.cpp index 26e959a18..3ab69b895 100644 --- a/dDashboardServer/routes/UgcRoutes.cpp +++ b/dDashboardServer/routes/UgcRoutes.cpp @@ -1,18 +1,15 @@ #include "UgcRoutes.h" #include -#include #include -#include - #include "CDClientDatabase.h" #include "Database.h" #include "UgcIconParams.h" #include "NifFile.h" -#include "TtlCache.h" #include "SettingsCatalog.h" #include "SettingsHistory.h" +#include "UgcFetch.h" #include "UgcKeys.h" #include "Workers.h" #include "Game.h" @@ -24,6 +21,7 @@ #include "eHTTPMethod.h" using namespace RouteUtils; +using namespace UgcFetch; namespace { constexpr uint32_t PAGE_SIZE = 50; @@ -33,91 +31,6 @@ namespace { std::optional ParseState(const std::string& text) { return IUgc::ParseProcessState(text); } - /** - * The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files - * it made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Fetches - * run on worker threads (Workers::Reply) with the URL worked out on the web thread; small answers are kept briefly. - */ - struct Fetched { - long status{}; // HTTP status, 0 when the UGC server didn't answer - std::string body; - std::string error; - }; - - std::mutex g_CacheMutex; - TtlCache> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024); - TtlCache> g_StatusCache(std::chrono::seconds(2), 1024 * 1024); - - constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024; - - size_t Collect(char* data, size_t size, size_t count, void* userData) { - auto* out = static_cast(userData); - if (out->size() + size * count > MAX_FETCH_BYTES) return 0; - out->append(data, size * count); - return size * count; - } - - // Web thread: where the dashboard reaches the UGC server - std::string InternalUrl() { - auto url = Game::config->GetValue("ugc_internal_url"); - if (url.empty()) url = "http://127.0.0.1:2008"; - while (url.ends_with('/')) url.pop_back(); - return url; - } - - // Any thread: a GET to the UGC server - std::shared_ptr Get(const std::string& url) { - auto out = std::make_shared(); - CURL* curl = curl_easy_init(); - if (!curl) { - out->error = "could not start a request"; - return out; - } - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard"); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); - const auto code = curl_easy_perform(curl); - if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); - else out->error = curl_easy_strerror(code); - curl_easy_cleanup(curl); - return out; - } - - // Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks (the master password) - std::shared_ptr AdminPost(const std::string& url, const std::string& key, const std::string& body) { - auto out = std::make_shared(); - CURL* curl = curl_easy_init(); - if (!curl) { - out->error = "could not start a request"; - return out; - } - curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json"); - headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str()); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https"); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(body.size())); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body); - const auto code = curl_easy_perform(curl); - if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status); - else out->error = curl_easy_strerror(code); - curl_slist_free_all(headers); - curl_easy_cleanup(curl); - return out; - } - // Web thread: the key the UGC server's /admin routes want std::string AdminKey() { const auto master = Database::Get()->GetMasterInfo(); @@ -194,27 +107,6 @@ namespace { return nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(*stored))); } - // Any thread: Get, through the cache (answers that aren't a file being made are kept) - std::shared_ptr CachedGet(const std::string& url, bool status) { - { - std::lock_guard lock(g_CacheMutex); - if (auto hit = (status ? g_StatusCache : g_Cache).Get(url)) return *hit; - } - auto fetched = Get(url); - if (fetched->status == 200 || fetched->status == 404) { - std::lock_guard lock(g_CacheMutex); - (status ? g_StatusCache : g_Cache).Put(url, fetched, fetched->body.size() + 64); - } - return fetched; - } - - void FetchError(HTTPReply& reply, const Fetched& fetched, const std::string& url) { - if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment"); - if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has no such file"); - if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched.error + ")"); - JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status)); - } - const std::set FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" }; @@ -293,7 +185,7 @@ namespace UgcRoutes { [](HTTPReply& reply, const HTTPContext& context) { const auto base = InternalUrl(); Workers::Reply(reply, context, false, [base](HTTPReply& out) { - const auto fetched = CachedGet(base + "/status", true); + const auto fetched = CachedGet(base + "/status", eCache::STATUS); if (fetched->status != 200) { JsonReply(out, eHTTPStatusCode::OK, { { "success", false }, { "url", base }, { "error", fetched->status == 0 ? fetched->error : "answered " + std::to_string(fetched->status) } }); return; @@ -315,8 +207,8 @@ namespace UgcRoutes { if (!kind || !id || !FILES.contains(file)) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid kind, id or file"); const auto url = InternalUrl() + "/files/" + *kind + "/" + std::to_string(*id) + "/" + file; Workers::Reply(reply, context, false, [url, file](HTTPReply& out) { - const auto fetched = CachedGet(url, false); - if (fetched->status != 200) return FetchError(out, *fetched, url); + const auto fetched = CachedGet(url); + if (fetched->status != 200) return ReplyError(out, *fetched, url); out.status = eHTTPStatusCode::OK; out.contentType = file.ends_with(".png") ? eContentType::IMAGE_PNG : file.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM; out.message = fetched->body; @@ -336,8 +228,8 @@ namespace UgcRoutes { const std::string file = std::string(previous ? "previous." : "") + (baked ? "model.nif" : "model.noao.nif"); const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/" + file; Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) { - const auto fetched = CachedGet(url, false); - if (fetched->status != 200) return FetchError(out, *fetched, url); + const auto fetched = CachedGet(url); + if (fetched->status != 200) return ReplyError(out, *fetched, url); std::string error; const auto model = NifFile::Parse(fetched->body, lod, error); if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error);