izebra_projects/fan_speed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alarm on cooling that cannot keep up, not on load

Browse Source 
A rising temperature is what a busy machine looks like; the previous
trend alarm would have mailed continuously through a multi-hour zpool
migration. trend_verdict() now alarms only when the fans are already at
maximum and it is still climbing, or when temperature and fan speed are
both rising and the rise is not decelerating.

Tells a plateau from a runaway by comparing the first half of the window
against the second - a load step settles once the fans catch up, a
failing fan does not. Verified quiet against a live migration at load
average 18 with CPU at 68c.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
2026-08-28 22:00:17 -07:00
parent 86f3b0c323
commit 80fc40a1ff
3 changed files with 93 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
fan_speed.sh
View File

@@ -117,6 +117,23 @@ selftest() {
# leak is what let the setpoint drift 32 -> 36 -> 31 on iz-pve0.
check "re-assert re-pushes held value" 32 "$(next_fan_speed 34 32)"
echo "trend verdict (load is not a fault - only cooling that cannot keep up is):"
TREND_SAMPLES=5
HIGH_FAN_SPEED=50
mkwindow() { : > "$TREND_FILE"; for pair in $@; do echo "0,${pair%%:*},0,${pair%%:*},${pair##*:}" >> "$TREND_FILE"; done; }
mkwindow 50:20 50:20 51:20 50:20 50:20
check "flat temps, fans steady" quiet "$(trend_verdict 2 5)"
mkwindow 50:50 54:50 57:50 59:50 60:50
check "climbing with fans flat out" no_headroom "$(trend_verdict 2 5)"
mkwindow 50:20 53:25 56:30 59:35 62:40
check "climbing as fast as the fans ramp" not_converging "$(trend_verdict 2 5)"
mkwindow 50:20 55:30 58:35 59:35 59:35
check "load step that plateaued" quiet "$(trend_verdict 2 5)"
mkwindow 50:20 54:20 57:20 59:20 60:20
check "climbing but fans never moved" quiet "$(trend_verdict 2 5)"
: > "$TREND_FILE"; echo "0,50,0,50,20" >> "$TREND_FILE"
check "partial window stays silent" quiet "$(trend_verdict 2 5)"
echo "ramp/alarm ordering (full cooling must arrive no later than the alarm):"
ramp_ordering_ok 8 5 && check "full at limit-8, alarm at limit-5" ok ok || check "full at limit-8, alarm at limit-5" ok bad
ramp_ordering_ok 5 5 && check "full and alarm coincide" ok ok || check "full and alarm coincide" ok bad
@@ -312,14 +329,21 @@ check_alarms() {
clear_alert gpu "GPU temperature normal on $(hostname)"
fi
# Climbing steadily while still under every threshold - a dying fan or a
# blocked intake looks exactly like this long before anything crosses a limit.
local cpu_rise disk_rise
# Climbing is only a fault if the fans are not answering it. A busy machine
# heats up and plateaus; a dying fan or blocked intake keeps climbing. See
# trend_verdict() - this stays quiet through hours of legitimate load.
local cpu_verdict disk_verdict cpu_rise disk_rise minutes
cpu_verdict=$(trend_verdict 2 5)
disk_verdict=$(trend_verdict 4 5)
cpu_rise=$(rise_over_window 2)
disk_rise=$(rise_over_window 4)
if [ "$cpu_rise" -ge "$TREND_RISE_ALARM" ] || [ "$disk_rise" -ge "$TREND_RISE_ALARM" ]; then
raise_alert trend "Temperature climbing on $(hostname)" \
"Over the last $((TREND_SAMPLES * CHECK_INTERVAL / 60)) minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c. Nothing has crossed a threshold yet. Check airflow and fans."
minutes=$((TREND_SAMPLES * CHECK_INTERVAL / 60))
if [ "$cpu_verdict" = no_headroom ] || [ "$disk_verdict" = no_headroom ]; then
raise_alert trend "Out of cooling headroom on $(hostname)" \
"Fans are at ${HIGH_FAN_SPEED}% and temperatures are still climbing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c ($HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c of ${HOTTEST_DISK_LIMIT}c). There is no cooling left to apply."
elif [ "$cpu_verdict" = not_converging ] || [ "$disk_verdict" = not_converging ]; then
raise_alert trend "Temperature outrunning the fans on $(hostname)" \
"Temperatures are climbing and the fans are ramping with them, but the rise is not slowing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c, fans now ${APPLIED_FAN_SPEED}%. Check airflow, intake and fan health."
else
clear_alert trend "Temperature stabilised on $(hostname)"
fi