diff --git a/README.md b/README.md index 100a56d..1031214 100644 --- a/README.md +++ b/README.md @@ -92,14 +92,26 @@ Email via `mail` to `ALERT_EMAIL`, which the host's postfix relays. Four conditi | Alarm | Fires when | |---|---| | `disk_temp` | A drive is within `DISK_ALARM_OFFSET` (5c) of its own limit | -| `trend` | CPU or hottest disk has climbed `TREND_RISE_ALARM` (8c) across the window while still under every threshold | +| `trend` | Temperatures climbing **and the fans cannot keep up** — see below | | `smart` | `smartctl -H` reports anything other than PASSED/OK | | `cpu` / `gpu` | Threshold crossed; `cpu` also means fan control was handed back to Dell's profile | | `disk_count` | Fewer drives answered than were present at startup | -**The trend alarm is the one worth having.** A dying fan or a blocked intake shows up as -a steady climb long before anything crosses a threshold — by the time an absolute alarm -fires you have already been running hot for hours. +**The trend alarm is the one worth having**, but a rising temperature is not by itself a +fault — it is what a busy machine looks like, and a multi-hour zpool migration will do it +all afternoon. What matters is whether the fans are answering. `trend_verdict()` returns +one of three things: + +| Verdict | Meaning | Alarms | +|---|---|---| +| `no_headroom` | Climbing while the fans are already at `HIGH_FAN_SPEED` | yes — nothing left to give | +| `not_converging` | Climbing, fans ramping with it, and the rise is *not slowing down* | yes — cooling is losing ground | +| `quiet` | Anything else, including a large climb the fans absorbed and that has plateaued | no — that is just load | + +It tells a plateau from a runaway by comparing the first half of the window against the +second: a normal load step decelerates once the fans catch up, a failing fan or a blocked +intake does not. A dying fan still gets caught long before an absolute threshold, without +mailing you every time the machine gets busy. Every alarm is **rate limited per key** with a one hour cooldown, and sends a single recovery notice when it clears. On a 10s loop an un-throttled alarm sends 360 emails an diff --git a/fan_speed.sh b/fan_speed.sh index 2f325ce..2ea7472 100755 --- a/fan_speed.sh +++ b/fan_speed.sh @@ -117,6 +117,23 @@ selftest() { # leak is what let the setpoint drift 32 -> 36 -> 31 on iz-pve0. check "re-assert re-pushes held value" 32 "$(next_fan_speed 34 32)" + echo "trend verdict (load is not a fault - only cooling that cannot keep up is):" + TREND_SAMPLES=5 + HIGH_FAN_SPEED=50 + mkwindow() { : > "$TREND_FILE"; for pair in $@; do echo "0,${pair%%:*},0,${pair%%:*},${pair##*:}" >> "$TREND_FILE"; done; } + mkwindow 50:20 50:20 51:20 50:20 50:20 + check "flat temps, fans steady" quiet "$(trend_verdict 2 5)" + mkwindow 50:50 54:50 57:50 59:50 60:50 + check "climbing with fans flat out" no_headroom "$(trend_verdict 2 5)" + mkwindow 50:20 53:25 56:30 59:35 62:40 + check "climbing as fast as the fans ramp" not_converging "$(trend_verdict 2 5)" + mkwindow 50:20 55:30 58:35 59:35 59:35 + check "load step that plateaued" quiet "$(trend_verdict 2 5)" + mkwindow 50:20 54:20 57:20 59:20 60:20 + check "climbing but fans never moved" quiet "$(trend_verdict 2 5)" + : > "$TREND_FILE"; echo "0,50,0,50,20" >> "$TREND_FILE" + check "partial window stays silent" quiet "$(trend_verdict 2 5)" + echo "ramp/alarm ordering (full cooling must arrive no later than the alarm):" ramp_ordering_ok 8 5 && check "full at limit-8, alarm at limit-5" ok ok || check "full at limit-8, alarm at limit-5" ok bad ramp_ordering_ok 5 5 && check "full and alarm coincide" ok ok || check "full and alarm coincide" ok bad @@ -312,14 +329,21 @@ check_alarms() { clear_alert gpu "GPU temperature normal on $(hostname)" fi - # Climbing steadily while still under every threshold - a dying fan or a - # blocked intake looks exactly like this long before anything crosses a limit. - local cpu_rise disk_rise + # Climbing is only a fault if the fans are not answering it. A busy machine + # heats up and plateaus; a dying fan or blocked intake keeps climbing. See + # trend_verdict() - this stays quiet through hours of legitimate load. + local cpu_verdict disk_verdict cpu_rise disk_rise minutes + cpu_verdict=$(trend_verdict 2 5) + disk_verdict=$(trend_verdict 4 5) cpu_rise=$(rise_over_window 2) disk_rise=$(rise_over_window 4) - if [ "$cpu_rise" -ge "$TREND_RISE_ALARM" ] || [ "$disk_rise" -ge "$TREND_RISE_ALARM" ]; then - raise_alert trend "Temperature climbing on $(hostname)" \ - "Over the last $((TREND_SAMPLES * CHECK_INTERVAL / 60)) minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c. Nothing has crossed a threshold yet. Check airflow and fans." + minutes=$((TREND_SAMPLES * CHECK_INTERVAL / 60)) + if [ "$cpu_verdict" = no_headroom ] || [ "$disk_verdict" = no_headroom ]; then + raise_alert trend "Out of cooling headroom on $(hostname)" \ + "Fans are at ${HIGH_FAN_SPEED}% and temperatures are still climbing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c ($HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c of ${HOTTEST_DISK_LIMIT}c). There is no cooling left to apply." + elif [ "$cpu_verdict" = not_converging ] || [ "$disk_verdict" = not_converging ]; then + raise_alert trend "Temperature outrunning the fans on $(hostname)" \ + "Temperatures are climbing and the fans are ramping with them, but the rise is not slowing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c, fans now ${APPLIED_FAN_SPEED}%. Check airflow, intake and fan health." else clear_alert trend "Temperature stabilised on $(hostname)" fi diff --git a/monitor.sh b/monitor.sh index ef7bdda..b3ec59b 100644 --- a/monitor.sh +++ b/monitor.sh @@ -258,9 +258,55 @@ record_sample() { fi } +# Is the cooling losing? A rising temperature on its own is not a fault - it is +# what a busy machine looks like, and a zpool migration will do it for hours. +# What matters is whether the fans are answering it: +# +# no_headroom temperature climbing while the fans are already flat out. +# Unambiguous: there is nothing left to give. +# not_converging temperature climbing, fans climbing with it, and the rise is +# not slowing down. Cooling is responding but losing ground. +# quiet anything else, including a big climb that the fans absorbed +# and that has since plateaued. That is just load. +# +# Compares the first half of the window against the second to tell a plateau +# from a runaway - a normal load step decelerates, a cooling failure does not. +# +# trend_verdict +trend_verdict() { + local tcol=$1 fcol=$2 lines window oldest mid newest fan_old fan_new first second + lines=$(wc -l < "$TREND_FILE" 2>/dev/null || echo 0) + if [ "$lines" -lt "$TREND_SAMPLES" ]; then echo quiet; return; fi + window=$(tail -n "$TREND_SAMPLES" "$TREND_FILE") + + oldest=$(printf '%s\n' "$window" | head -1 | cut -d, -f"$tcol") + mid=$(printf '%s\n' "$window" | sed -n "$(( (TREND_SAMPLES + 1) / 2 ))p" | cut -d, -f"$tcol") + newest=$(printf '%s\n' "$window" | tail -1 | cut -d, -f"$tcol") + fan_old=$(printf '%s\n' "$window" | head -1 | cut -d, -f"$fcol") + fan_new=$(printf '%s\n' "$window" | tail -1 | cut -d, -f"$fcol") + [ -n "$oldest" ] && [ -n "$mid" ] && [ -n "$newest" ] || { echo quiet; return; } + + # Not climbing meaningfully - nothing to say either way. + [ $((newest - oldest)) -ge "$TREND_RISE_ALARM" ] || { echo quiet; return; } + + # Climbing with the fans already flat out. + if [ "$fan_new" -ge "$HIGH_FAN_SPEED" ]; then echo no_headroom; return; fi + + # Climbing, fans climbing too, and the second half of the window rose at least + # as fast as the first - it is not settling. + first=$((mid - oldest)) + second=$((newest - mid)) + if [ "$fan_new" -gt "$fan_old" ] && [ "$second" -ge "$first" ]; then + echo not_converging; return + fi + + echo quiet +} + # rise_over_window -> degrees climbed from the oldest sample in the # window to the newest. 0 until there is a full window of history, so a restart -# cannot alarm on a partial series. +# cannot alarm on a partial series. Reported in alarm text; the decision to +# alarm belongs to trend_verdict. # Columns: 2 cpu, 3 gpu, 4 disk, 5 fan. rise_over_window() { local column=$1 oldest newest lines