#include "UgcStorage.h" #include #include #include #include #include "ZCompression.h" namespace { constexpr std::array KNOWN_FILES = { "model.nif", "model.nif.gz", "model.nif.checksum", "model.nif.sd0", "model.lxfml.gz", "model.lxfml.checksum", "icon.dds.gz", "icon.dds.checksum", "icon.dds.sd0", "icon.png", "model.noao.nif", "model.noao.nif.gz", "stats.json", "combo.json", "previous.icon.png", "previous.model.nif.gz", "previous.stats.json", }; // What of the files made before is kept when an item is made again, for comparing (as previous.) constexpr std::array KEPT_FILES = { "icon.png", "model.nif.gz", "stats.json" }; const char* KindFolder(UgcStorage::Kind kind) { return kind == UgcStorage::Kind::MODEL ? "models" : "modular"; } std::string RandomSuffix() { static thread_local std::mt19937_64 random{ std::random_device{}() }; return std::to_string(random()); } } UgcStorage::UgcStorage(std::filesystem::path root) : m_Root(std::move(root)) {} bool UgcStorage::IsKnownFile(const std::string& name) { return std::find(KNOWN_FILES.begin(), KNOWN_FILES.end(), name) != KNOWN_FILES.end(); } std::filesystem::path UgcStorage::Folder(Kind kind, LWOOBJID id) const { const auto id64 = static_cast(id); return m_Root / KindFolder(kind) / std::to_string(id64 % 1000) / std::to_string(id64); } std::optional UgcStorage::File(Kind kind, LWOOBJID id, const std::string& name) const { if (!IsKnownFile(name)) return std::nullopt; auto path = Folder(kind, id) / name; std::error_code error; if (!std::filesystem::is_regular_file(path, error)) return std::nullopt; return path; } std::optional UgcStorage::Write(Kind kind, LWOOBJID id, const Files& files, std::string& error) const { const auto folder = Folder(kind, id); const auto temporary = folder.parent_path() / (".tmp-" + folder.filename().string() + "-" + RandomSuffix()); std::error_code code; std::filesystem::create_directories(temporary, code); if (code) { error = "could not create " + temporary.string() + ": " + code.message(); return std::nullopt; } uint64_t bytes = 0; for (const auto& [name, data] : files) { std::ofstream out(temporary / name, std::ios::binary | std::ios::trunc); out.write(data.data(), static_cast(data.size())); if (!out) { error = "could not write " + (temporary / name).string(); std::filesystem::remove_all(temporary, code); return std::nullopt; } bytes += data.size(); } // Keep the last version's previews to compare with for (const auto* name : KEPT_FILES) { if (files.contains(name)) { std::filesystem::copy_file(folder / name, temporary / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code); if (!code) bytes += std::filesystem::file_size(temporary / (std::string("previous.") + name), code); code.clear(); } } // Swap the old folder out and the new one in; the old one is deleted after const auto old = folder.parent_path() / (".old-" + folder.filename().string() + "-" + RandomSuffix()); const bool hadOld = std::filesystem::exists(folder, code); if (hadOld) std::filesystem::rename(folder, old, code); std::filesystem::rename(temporary, folder, code); if (code) { error = "could not move the files into " + folder.string() + ": " + code.message(); std::filesystem::remove_all(temporary, code); return std::nullopt; } if (hadOld) std::filesystem::remove_all(old, code); return bytes; } std::optional UgcStorage::Update(Kind kind, LWOOBJID id, const Files& files, std::string& error) const { const auto folder = Folder(kind, id); std::error_code code; if (!std::filesystem::is_directory(folder, code)) { error = "no files to update in " + folder.string(); return std::nullopt; } uint64_t bytes = 0; for (const auto* name : KEPT_FILES) { if (files.contains(name) && std::filesystem::exists(folder / name, code)) { std::filesystem::copy_file(folder / name, folder / (std::string("previous.") + name), std::filesystem::copy_options::overwrite_existing, code); code.clear(); } } for (const auto& [name, data] : files) { if (!IsKnownFile(name)) continue; const auto aside = folder / (".tmp-" + name + "-" + RandomSuffix()); { std::ofstream out(aside, std::ios::binary | std::ios::trunc); out.write(data.data(), static_cast(data.size())); if (!out) { error = "could not write " + aside.string(); std::filesystem::remove(aside, code); return std::nullopt; } } std::filesystem::rename(aside, folder / name, code); if (code) { error = "could not replace " + (folder / name).string() + ": " + code.message(); std::filesystem::remove(aside, code); return std::nullopt; } bytes += data.size(); } return bytes; } void UgcStorage::Remove(Kind kind, LWOOBJID id) const { std::error_code error; std::filesystem::remove_all(Folder(kind, id), error); } void UgcStorage::Touch(Kind kind, LWOOBJID id) const { std::error_code error; std::filesystem::last_write_time(Folder(kind, id), std::filesystem::file_time_type::clock::now(), error); } std::vector UgcStorage::List() const { std::vector entries; std::error_code error; for (const auto kind : { Kind::MODEL, Kind::MODULAR }) { const auto base = m_Root / KindFolder(kind); for (std::filesystem::directory_iterator bucket(base, error), end; !error && bucket != end; bucket.increment(error)) { if (!bucket->is_directory(error)) continue; for (std::filesystem::directory_iterator item(bucket->path(), error), itemEnd; !error && item != itemEnd; item.increment(error)) { const auto name = item->path().filename().string(); if (name.empty() || name[0] == '.' || !item->is_directory(error)) continue; Entry entry; entry.kind = kind; try { entry.id = static_cast(std::stoull(name)); } catch (...) { continue; } entry.used = std::filesystem::last_write_time(item->path(), error); for (std::filesystem::directory_iterator file(item->path(), error), fileEnd; !error && file != fileEnd; file.increment(error)) { if (file->is_regular_file(error)) entry.bytes += file->file_size(error); } entries.push_back(entry); } error.clear(); } error.clear(); } return entries; } std::vector UgcStorage::Evict(uint64_t maxBytes) const { auto entries = List(); uint64_t total = 0; for (const auto& entry : entries) total += entry.bytes; std::vector removed; if (total <= maxBytes) return removed; std::sort(entries.begin(), entries.end(), [](const Entry& a, const Entry& b) { return a.used < b.used; }); for (const auto& entry : entries) { if (total <= maxBytes) break; Remove(entry.kind, entry.id); total -= std::min(total, entry.bytes); removed.push_back(entry); } return removed; } std::optional UgcStorage::ReadNif(Kind kind, LWOOBJID id, const std::string& name) const { const auto read = [](const std::filesystem::path& path) -> std::optional { std::ifstream in(path, std::ios::binary); if (!in) return std::nullopt; return std::string(std::istreambuf_iterator(in), {}); }; if (const auto packed = File(kind, id, name + ".gz")) { const auto data = read(*packed); return data ? ZCompression::Gunzip(*data) : std::nullopt; } if (const auto plain = File(kind, id, name)) return read(*plain); return std::nullopt; }