fix(ugc): stopping the UGC server cancels the models being made

Stopping waited for every worker to finish its model, and with the path
traced hidden-face removal a big model takes minutes, so a restart hung.
UgcThrottle::Cancel(true) (set by UgcProcessor::Stop) makes Checkpoint,
which the long loops already call, throw UgcThrottle::Cancelled; the worker
abandons the job without writing or recording anything, so its row stays
waiting and is made again after the restart. The throttle's sleeps wake to
check it. Previews answer 503.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dUgcServer/UgcProcessor.cpp

@@ -133,6 +133,7 @@ void UgcProcessor::Start() {
 	}
 	m_NextEviction = std::chrono::steady_clock::now();
 	m_Stopping = false;
+	UgcThrottle::Cancel(false);
 	for (size_t i = 0; i < std::max<size_t>(m_Config.threads, 1); i++) m_Threads.emplace_back(&UgcProcessor::Worker, this);
 	LOG("UGC processing started with %zu worker(s), %llu MB stored", m_Threads.size(), static_cast<unsigned long long>(m_StoredBytes / (1024 * 1024)));
 }
@@ -143,6 +144,9 @@ void UgcProcessor::Stop() {
 		m_Stopping = true;
 		m_Jobs.clear();
 	}
+	// Jobs being made stop at their next checkpoint instead of finishing (a big model can take minutes); their rows stay
+	// waiting and are made again after the restart
+	UgcThrottle::Cancel(true);
 	m_Wake.notify_all();
 	for (auto& thread : m_Threads) {
 		if (thread.joinable()) thread.join();
@@ -256,6 +260,10 @@ void UgcProcessor::Worker() {
 					reply.contentType = eContentType::TEXT_PLAIN;
 					reply.message = outcome.error;
 				}
+			} catch (const UgcThrottle::Cancelled&) {
+				reply.status = eHTTPStatusCode::SERVICE_UNAVAILABLE;
+				reply.contentType = eContentType::TEXT_PLAIN;
+				reply.message = "the UGC server is stopping";
 			} catch (const std::exception& ex) {
 				reply.status = eHTTPStatusCode::INTERNAL_SERVER_ERROR;
 				reply.contentType = eContentType::TEXT_PLAIN;
@@ -288,6 +296,12 @@ void UgcProcessor::Worker() {
 					? UgcJobs::ProcessModel(job.blob, m_Library, settings, static_cast<uint64_t>(job.id), job.iconValues)
 					: UgcJobs::ProcessModular(job.modular, m_Library.GetResPath(), settings);
 			}
+		} catch (const UgcThrottle::Cancelled&) {
+			// Stopping: abandoned, not failed; nothing is written or recorded
+			std::lock_guard lock(m_Mutex);
+			m_Active--;
+			m_MemoryInUse -= std::min(m_MemoryInUse, job.memory);
+			continue;
 		} catch (const std::exception& ex) {
 			done.outcome.ok = false;
 			done.outcome.error = std::string("crashed: ") + ex.what();
@@ -315,7 +329,11 @@ void UgcProcessor::Worker() {
 		done.outcome.files.clear();
 		done.milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
 		done.cpuMilliseconds = std::max(0.0, UgcThrottle::ThreadCpuSeconds() - cpuStart) * 1000.0;
-		UgcThrottle::Checkpoint();
+		try {
+			UgcThrottle::Checkpoint();
+		} catch (const UgcThrottle::Cancelled&) {
+			// Stopping: the job is done already, so it's still recorded
+		}
 
 		{
 			std::lock_guard lock(m_Mutex);

dUgcServer/UgcThrottle.cpp

@@ -14,6 +14,7 @@ namespace {
 	constexpr double MIN_ACCOUNT_SECONDS = 0.005;
 
 	std::atomic<double> g_Budget{ 0.0 };
+	std::atomic<bool> g_Cancel{ false };
 	std::mutex g_Mutex;
 	double g_Balance = BURST_SECONDS; // CPU seconds that may still be used
 	std::chrono::steady_clock::time_point g_Refilled = std::chrono::steady_clock::now();
@@ -58,7 +59,11 @@ namespace UgcThrottle {
 		t_LastCpu = ThreadCpuSeconds();
 	}
 
+	void Cancel(const bool cancel) { g_Cancel = cancel; }
+	bool IsCancelled() { return g_Cancel; }
+
 	void Checkpoint() {
+		if (g_Cancel) throw Cancelled{};
 		const double budget = g_Budget;
 		if (budget <= 0.0) return;
 		const double cpu = ThreadCpuSeconds();
@@ -80,7 +85,12 @@ namespace UgcThrottle {
 		wait = std::min(wait, 5.0);
 		g_SleptMs += static_cast<uint64_t>(wait * 1000.0);
 		g_LastSleep = UnixMs();
-		std::this_thread::sleep_for(std::chrono::duration<double>(wait));
+		// In short sleeps, so a cancel isn't held up by a long wait
+		const auto until = std::chrono::steady_clock::now() + std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::duration<double>(wait));
+		while (std::chrono::steady_clock::now() < until) {
+			if (g_Cancel) throw Cancelled{};
+			std::this_thread::sleep_for(std::min<std::chrono::steady_clock::duration>(until - std::chrono::steady_clock::now(), std::chrono::milliseconds(100)));
+		}
 		// Time asleep costs no CPU; don't count this call's own bookkeeping twice
 		t_LastCpu = ThreadCpuSeconds();
 	}

dUgcServer/UgcThrottle.h

@@ -14,9 +14,16 @@ namespace UgcThrottle {
 	void SetBudget(double cpus);
 	double GetBudget();
 
-	// Account the calling thread's CPU time and sleep when over the budget
+	// Account the calling thread's CPU time and sleep when over the budget. Throws Cancelled once Cancel(true) was
+	// called, so a job stops in moments when the server shuts down.
 	void Checkpoint();
 
+	// Thrown by Checkpoint while cancelled: the job is abandoned, not failed (its row stays waiting)
+	struct Cancelled {};
+	// Stop (true) or allow (false) every job's work: set when the server stops, cleared when it starts
+	void Cancel(bool cancel);
+	bool IsCancelled();
+
 	// Start accounting on this thread from now (a worker starting a job), so time spent idle isn't counted
 	void Begin();