Mirrored_Repos/DarkflameServer
1
0
Fork 0
mirror of https://github.com/DarkflameUniverse/DarkflameServer.git synced 2026-10-02 10:53:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(dashboard): one helper for fetching from the UGC server

Browse Source 
The /ugc page's routes and the UGC links' icon and mesh routes each had
their own copy of the curl request and its short cache. Both use
UgcFetch now: the URL is worked out on the web thread, the request and
cache run on worker threads. Players' errors still don't name the
internal URL.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Aaron Kimbrell
2026-09-27 14:58:50 -05:00
parent 8ab7e496b0
commit f65307992e
5 changed files with 160 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dDashboardServer/routes/CMakeLists.txt
View File

@@ -58,6 +58,7 @@ set(DASHBOARDROUTES_SOURCES
"ContrabandRoutes.cpp"
"UgcRoutes.cpp"
"UgcLinks.cpp"
"UgcFetch.cpp"
"PropertyRentRoutes.cpp"
"PasswordRecovery.cpp"
"PrometheusMetrics.cpp"

102
dDashboardServer/routes/UgcFetch.cpp Normal file
View File

@@ -0,0 +1,102 @@
#include "UgcFetch.h"
#include <chrono>
#include <mutex>
#include <curl/curl.h>
#include "dConfig.h"
#include "Game.h"
#include "RouteUtils.h"
#include "TtlCache.h"
namespace {
std::mutex g_CacheMutex;
TtlCache<std::string, std::shared_ptr<const UgcFetch::Fetched>> g_FileCache(std::chrono::seconds(15), 32 * 1024 * 1024);
TtlCache<std::string, std::shared_ptr<const UgcFetch::Fetched>> g_StatusCache(std::chrono::seconds(2), 1024 * 1024);
constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024;
size_t Collect(char* data, size_t size, size_t count, void* userData) {
auto* out = static_cast<std::string*>(userData);
if (out->size() + size * count > MAX_FETCH_BYTES) return 0;
out->append(data, size * count);
return size * count;
}
// Sends a request set up by setup (a GET unless it says otherwise)
template <typename Setup>
std::shared_ptr<const UgcFetch::Fetched> Perform(const std::string& url, long timeout, Setup&& setup) {
auto out = std::make_shared<UgcFetch::Fetched>();
CURL* curl = curl_easy_init();
if (!curl) {
out->error = "could not start a request";
return out;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body);
setup(curl);
const auto code = curl_easy_perform(curl);
if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status);
else out->error = curl_easy_strerror(code);
curl_easy_cleanup(curl);
return out;
}
}
namespace UgcFetch {
std::string InternalUrl() {
auto url = Game::config->GetValue("ugc_internal_url");
if (url.empty()) url = "http://127.0.0.1:2008";
while (url.ends_with('/')) url.pop_back();
return url;
}
std::shared_ptr<const Fetched> Get(const std::string& url) {
return Perform(url, 20L, [](CURL*) {});
}
std::shared_ptr<const Fetched> CachedGet(const std::string& url, eCache cache) {
auto& store = cache == eCache::STATUS ? g_StatusCache : g_FileCache;
{
std::lock_guard lock(g_CacheMutex);
if (auto hit = store.Get(url)) return *hit;
}
auto fetched = Get(url);
if (fetched->status == 200 || fetched->status == 404) {
std::lock_guard lock(g_CacheMutex);
store.Put(url, fetched, fetched->body.size() + 64);
}
return fetched;
}
std::shared_ptr<const Fetched> AdminPost(const std::string& url, const std::string& key, const std::string& body) {
curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json");
headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str());
auto out = Perform(url, 120L, [&](CURL* curl) {
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast<long>(body.size()));
});
curl_slist_free_all(headers);
return out;
}
void ReplyError(HTTPReply& reply, const Fetched& fetched, std::string_view shownUrl) {
using RouteUtils::JsonError;
if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment");
if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has not made it");
if (fetched.status == 0) {
const std::string at = shownUrl.empty() ? std::string() : " at " + std::string(shownUrl);
return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer" + at + " (" + fetched.error + ")");
}
JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status));
}
}

44
dDashboardServer/routes/UgcFetch.h Normal file
View File

@@ -0,0 +1,44 @@
#ifndef UGCFETCH_H
#define UGCFETCH_H
#include <memory>
#include <string>
#include <string_view>
struct HTTPReply;
/**
* The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files it
* made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Work out the
* URL (and anything else from the config or database) on the web thread; the requests themselves only touch curl and
* this file's cache, so they can run on worker threads (Workers::Reply).
*/
namespace UgcFetch {
struct Fetched {
long status{}; // HTTP status, 0 when the UGC server didn't answer
std::string body;
std::string error;
};
enum class eCache {
FILE, // files the UGC server made, kept 15 seconds
STATUS, // its status, kept 2 seconds
};
// Web thread: where the dashboard reaches the UGC server, without a trailing slash
std::string InternalUrl();
// Any thread: a GET to the UGC server, not cached
std::shared_ptr<const Fetched> Get(const std::string& url);
// Any thread: Get through a short cache (answers that aren't a file being made are kept)
std::shared_ptr<const Fetched> CachedGet(const std::string& url, eCache cache = eCache::FILE);
// Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks
std::shared_ptr<const Fetched> AdminPost(const std::string& url, const std::string& key, const std::string& body);
// Answers with why a fetch didn't give a file. The URL is only named when shown (not to players, who don't need it)
void ReplyError(HTTPReply& reply, const Fetched& fetched, std::string_view shownUrl = {});
}
#endif // !UGCFETCH_H

79
dDashboardServer/routes/UgcLinks.cpp
View File

@@ -2,11 +2,8 @@
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <curl/curl.h>
#include "Database.h"
#include "dConfig.h"
#include "eHTTPMethod.h"
@@ -14,83 +11,19 @@
#include "GeneralUtils.h"
#include "NifFile.h"
#include "RouteUtils.h"
#include "TtlCache.h"
#include "UgcFetch.h"
#include "UgcLookup.h"
#include "Workers.h"
using namespace RouteUtils;
using namespace UgcLookup;
using namespace UgcFetch;
namespace {
constexpr uint32_t SEARCH_LIMIT = 50;
// Creators whose inventories a search looks in for creations that are not placed or mailed
constexpr size_t MAX_INVENTORIES = 25;
// ---- The UGC server's files, fetched by the dashboard (as the /ugc page's routes do) ----
struct Fetched {
long status{}; // HTTP status, 0 when the UGC server didn't answer
std::string body;
std::string error;
};
std::mutex g_CacheMutex;
TtlCache<std::string, std::shared_ptr<const Fetched>> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024);
constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024;
size_t Collect(char* data, size_t size, size_t count, void* userData) {
auto* out = static_cast<std::string*>(userData);
if (out->size() + size * count > MAX_FETCH_BYTES) return 0;
out->append(data, size * count);
return size * count;
}
// Web thread
std::string InternalUrl() {
auto url = Game::config->GetValue("ugc_internal_url");
if (url.empty()) url = "http://127.0.0.1:2008";
while (url.ends_with('/')) url.pop_back();
return url;
}
// Any thread
std::shared_ptr<const Fetched> Fetch(const std::string& url) {
{
std::lock_guard lock(g_CacheMutex);
if (auto hit = g_Cache.Get(url)) return *hit;
}
auto out = std::make_shared<Fetched>();
CURL* curl = curl_easy_init();
if (!curl) {
out->error = "could not start a request";
return out;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body);
const auto code = curl_easy_perform(curl);
if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status);
else out->error = curl_easy_strerror(code);
curl_easy_cleanup(curl);
if (out->status == 200 || out->status == 404) {
std::lock_guard lock(g_CacheMutex);
g_Cache.Put(url, out, out->body.size() + 64);
}
return out;
}
void FetchError(HTTPReply& reply, const Fetched& fetched) {
if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment");
if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has not made it");
if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer (" + fetched.error + ")");
JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status));
}
// ---- Who may see a creation ----
@@ -298,8 +231,8 @@ namespace UgcLinks {
if (!MaySee(context, *kind, *id)) return JsonError(reply, eHTTPStatusCode::FORBIDDEN, "You may not view this creation");
const auto url = InternalUrl() + "/files/" + KindName(*kind) + "/" + std::to_string(*id) + "/icon.png";
Workers::Reply(reply, context, false, [url](HTTPReply& out) {
const auto fetched = Fetch(url);
if (fetched->status != 200) return FetchError(out, *fetched);
const auto fetched = CachedGet(url);
if (fetched->status != 200) return ReplyError(out, *fetched);
out.status = eHTTPStatusCode::OK;
out.contentType = eContentType::IMAGE_PNG;
out.message = fetched->body;
@@ -317,8 +250,8 @@ namespace UgcLinks {
const auto lod = std::min(GeneralUtils::TryParse<uint32_t>(QueryValue(context.queryString, "lod")).value_or(0), 3u);
const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/model.nif";
Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) {
const auto fetched = Fetch(url);
if (fetched->status != 200) return FetchError(out, *fetched);
const auto fetched = CachedGet(url);
if (fetched->status != 200) return ReplyError(out, *fetched);
std::string error;
const auto model = NifFile::Parse(fetched->body, lod, error);
if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error);

122
dDashboardServer/routes/UgcRoutes.cpp
View File

@@ -1,18 +1,15 @@
#include "UgcRoutes.h"
#include <memory>
#include <mutex>
#include <set>
#include <curl/curl.h>
#include "CDClientDatabase.h"
#include "Database.h"
#include "UgcIconParams.h"
#include "NifFile.h"
#include "TtlCache.h"
#include "SettingsCatalog.h"
#include "SettingsHistory.h"
#include "UgcFetch.h"
#include "UgcKeys.h"
#include "Workers.h"
#include "Game.h"
@@ -24,6 +21,7 @@
#include "eHTTPMethod.h"
using namespace RouteUtils;
using namespace UgcFetch;
namespace {
constexpr uint32_t PAGE_SIZE = 50;
@@ -33,91 +31,6 @@ namespace {
std::optional<IUgc::eProcessState> ParseState(const std::string& text) { return IUgc::ParseProcessState(text); }
/**
* The dashboard's own way to the UGC server (ugc_internal_url, normally the same machine): its status and the files
* it made are fetched here and handed to the browser, which may not be able to reach the UGC server at all. Fetches
* run on worker threads (Workers::Reply) with the URL worked out on the web thread; small answers are kept briefly.
*/
struct Fetched {
long status{}; // HTTP status, 0 when the UGC server didn't answer
std::string body;
std::string error;
};
std::mutex g_CacheMutex;
TtlCache<std::string, std::shared_ptr<const Fetched>> g_Cache(std::chrono::seconds(15), 32 * 1024 * 1024);
TtlCache<std::string, std::shared_ptr<const Fetched>> g_StatusCache(std::chrono::seconds(2), 1024 * 1024);
constexpr size_t MAX_FETCH_BYTES = 128 * 1024 * 1024;
size_t Collect(char* data, size_t size, size_t count, void* userData) {
auto* out = static_cast<std::string*>(userData);
if (out->size() + size * count > MAX_FETCH_BYTES) return 0;
out->append(data, size * count);
return size * count;
}
// Web thread: where the dashboard reaches the UGC server
std::string InternalUrl() {
auto url = Game::config->GetValue("ugc_internal_url");
if (url.empty()) url = "http://127.0.0.1:2008";
while (url.ends_with('/')) url.pop_back();
return url;
}
// Any thread: a GET to the UGC server
std::shared_ptr<const Fetched> Get(const std::string& url) {
auto out = std::make_shared<Fetched>();
CURL* curl = curl_easy_init();
if (!curl) {
out->error = "could not start a request";
return out;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "DarkflameServer-Dashboard");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body);
const auto code = curl_easy_perform(curl);
if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status);
else out->error = curl_easy_strerror(code);
curl_easy_cleanup(curl);
return out;
}
// Any thread: a POST of JSON to one of the UGC server's /admin routes, with the key it checks (the master password)
std::shared_ptr<const Fetched> AdminPost(const std::string& url, const std::string& key, const std::string& body) {
auto out = std::make_shared<Fetched>();
CURL* curl = curl_easy_init();
if (!curl) {
out->error = "could not start a request";
return out;
}
curl_slist* headers = curl_slist_append(nullptr, "Content-Type: application/json");
headers = curl_slist_append(headers, ("X-Ugc-Admin-Key: " + key).c_str());
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "http,https");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast<long>(body.size()));
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Collect);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out->body);
const auto code = curl_easy_perform(curl);
if (code == CURLE_OK) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &out->status);
else out->error = curl_easy_strerror(code);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return out;
}
// Web thread: the key the UGC server's /admin routes want
std::string AdminKey() {
const auto master = Database::Get()->GetMasterInfo();
@@ -194,27 +107,6 @@ namespace {
return nlohmann::json::parse(UgcIconParams::ToJson(UgcIconParams::Parse(*stored)));
}
// Any thread: Get, through the cache (answers that aren't a file being made are kept)
std::shared_ptr<const Fetched> CachedGet(const std::string& url, bool status) {
{
std::lock_guard lock(g_CacheMutex);
if (auto hit = (status ? g_StatusCache : g_Cache).Get(url)) return *hit;
}
auto fetched = Get(url);
if (fetched->status == 200 || fetched->status == 404) {
std::lock_guard lock(g_CacheMutex);
(status ? g_StatusCache : g_Cache).Put(url, fetched, fetched->body.size() + 64);
}
return fetched;
}
void FetchError(HTTPReply& reply, const Fetched& fetched, const std::string& url) {
if (fetched.status == 408) return JsonError(reply, eHTTPStatusCode::REQUEST_TIMEOUT, "The UGC server is making it; try again in a moment");
if (fetched.status == 404) return JsonError(reply, eHTTPStatusCode::NOT_FOUND, "The UGC server has no such file");
if (fetched.status == 0) return JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server doesn't answer at " + url + " (" + fetched.error + ")");
JsonError(reply, eHTTPStatusCode::BAD_GATEWAY, "The UGC server answered " + std::to_string(fetched.status));
}
const std::set<std::string> FILES = { "icon.png", "model.nif", "model.noao.nif", "stats.json", "combo.json",
"previous.icon.png", "previous.model.nif", "previous.model.noao.nif", "previous.stats.json" };
@@ -293,7 +185,7 @@ namespace UgcRoutes {
[](HTTPReply& reply, const HTTPContext& context) {
const auto base = InternalUrl();
Workers::Reply(reply, context, false, [base](HTTPReply& out) {
const auto fetched = CachedGet(base + "/status", true);
const auto fetched = CachedGet(base + "/status", eCache::STATUS);
if (fetched->status != 200) {
JsonReply(out, eHTTPStatusCode::OK, { { "success", false }, { "url", base }, { "error", fetched->status == 0 ? fetched->error : "answered " + std::to_string(fetched->status) } });
return;
@@ -315,8 +207,8 @@ namespace UgcRoutes {
if (!kind || !id || !FILES.contains(file)) return JsonError(reply, eHTTPStatusCode::BAD_REQUEST, "Invalid kind, id or file");
const auto url = InternalUrl() + "/files/" + *kind + "/" + std::to_string(*id) + "/" + file;
Workers::Reply(reply, context, false, [url, file](HTTPReply& out) {
const auto fetched = CachedGet(url, false);
if (fetched->status != 200) return FetchError(out, *fetched, url);
const auto fetched = CachedGet(url);
if (fetched->status != 200) return ReplyError(out, *fetched, url);
out.status = eHTTPStatusCode::OK;
out.contentType = file.ends_with(".png") ? eContentType::IMAGE_PNG : file.ends_with(".json") ? eContentType::APPLICATION_JSON : eContentType::APPLICATION_OCTET_STREAM;
out.message = fetched->body;
@@ -336,8 +228,8 @@ namespace UgcRoutes {
const std::string file = std::string(previous ? "previous." : "") + (baked ? "model.nif" : "model.noao.nif");
const auto url = InternalUrl() + "/files/model/" + std::to_string(*id) + "/" + file;
Workers::Reply(reply, context, false, [url, lod](HTTPReply& out) {
const auto fetched = CachedGet(url, false);
if (fetched->status != 200) return FetchError(out, *fetched, url);
const auto fetched = CachedGet(url);
if (fetched->status != 200) return ReplyError(out, *fetched, url);
std::string error;
const auto model = NifFile::Parse(fetched->body, lod, error);
if (!model) return JsonError(out, eHTTPStatusCode::UNPROCESSABLE_ENTITY, "The .nif can't be read: " + error);