80fc40a1ff
A rising temperature is what a busy machine looks like; the previous trend alarm would have mailed continuously through a multi-hour zpool migration. trend_verdict() now alarms only when the fans are already at maximum and it is still climbing, or when temperature and fan speed are both rising and the rise is not decelerating. Tells a plateau from a runaway by comparing the first half of the window against the second - a load step settles once the fans catch up, a failing fan does not. Verified quiet against a live migration at load average 18 with CPU at 68c. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
393 lines
18 KiB
Bash
Executable File
393 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Dell PowerEdge fan control driven by CPU, GPU and disk temperature, with
|
|
# trend tracking and email alarms.
|
|
#
|
|
# Usage: fan_speed.sh [run|once|disks|selftest] (default: run)
|
|
|
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
source "$SCRIPT_DIR/functions.sh"
|
|
source "$SCRIPT_DIR/monitor.sh"
|
|
|
|
# Trap the signals for container exit and run graceful_exit function
|
|
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
|
|
|
|
# Variable defintions
|
|
# LOW_FAN_SPEED = Lowest fan speed
|
|
# HIGH_FAN_SPEED = max fan speed. (0-100)
|
|
# IDRAC_HOST = local or IP of iDRAC
|
|
# IDRAC_USERNAME = username of iDRAC
|
|
# IDRAC_PASSWORD = password if iDRAC
|
|
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
|
|
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
|
|
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
|
|
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
|
|
|
|
IDRAC_HOST=local
|
|
IDRAC_USERNAME=root
|
|
IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin}
|
|
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
|
|
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
|
|
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
|
|
# Per-chassis: the PWM that matches what Dell's own profile runs at idle. The
|
|
# PWM->RPM relationship is chassis specific, so this is measured, not guessed -
|
|
# see "Calibrating a host" in the README. 18/50 suits an R730xd that idles at
|
|
# 48c; an R720xd idling at 60c needs a floor of 28 to match stock.
|
|
LOW_FAN_SPEED=${LOW_FAN_SPEED:-18}
|
|
HIGH_FAN_SPEED=${HIGH_FAN_SPEED:-50}
|
|
TABLE_HEADER_PRINT_INTERVAL=2
|
|
|
|
# The CPU ramp is derived from the CPU sensor's own upper-non-critical
|
|
# threshold as reported by iDRAC, exactly like the disks are derived from
|
|
# theirs. A fixed 45c start was calibrated on a chassis that idles at 48c; on
|
|
# one that idles at 58c it sat 40% up the ramp doing nothing useful, and ran
|
|
# ~10% more airflow than Dell's own profile chose at the same temperature
|
|
# (6878 vs 6240 RPM measured over 8 minutes on an R720xd).
|
|
#
|
|
# Offsets chosen so a 58c idle lands near what Dell picks. Both an R720xd and
|
|
# an R730xd report upper-non-critical 77c, so this resolves to a 53->69c ramp.
|
|
CPU_RAMP_LOW_OFFSET=${CPU_RAMP_LOW_OFFSET:-24}
|
|
CPU_RAMP_HIGH_OFFSET=${CPU_RAMP_HIGH_OFFSET:-8}
|
|
CPU_LIMIT_FALLBACK=${CPU_LIMIT_FALLBACK:-77}
|
|
|
|
# Disk thresholds, alarm routing and trend window all live in monitor.sh.
|
|
|
|
# iDRAC's declared upper-non-critical temperature for the CPU sensor.
|
|
cpu_upper_non_critical() {
|
|
ipmitool -I $IDRAC_LOGIN_STRING sdr get "Temp" 2>/dev/null |
|
|
awk -F: '/Upper non-critical/ {gsub(/[^0-9.]/,"",$2); print int($2); exit}'
|
|
}
|
|
|
|
#######################################
|
|
# Self test - pure logic only, no IPMI and no drives touched.
|
|
selftest() {
|
|
local tmp; tmp=$(mktemp -d)
|
|
export STATE_DIR="$tmp/state" TREND_FILE="$tmp/temps.csv" LOG_FILE="$tmp/log" DRY_RUN=1
|
|
local fail=0
|
|
check() { # check <description> <expected> <actual>
|
|
if [ "$2" = "$3" ]; then echo " ok $1"; else echo " FAIL $1: expected '$2' got '$3'"; fail=1; fi
|
|
}
|
|
|
|
echo "temperature parsing:"
|
|
check "SATA Temperature_Celsius" 35 \
|
|
"$(printf '194 Temperature_Celsius 0x0022 115 099 000 Old_age Always - 35\n' | parse_disk_temperature)"
|
|
check "Seagate Airflow_Temperature_Cel" 41 \
|
|
"$(printf '190 Airflow_Temperature_Cel 0x0032 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
|
|
check "Intel Temperature_Internal" 46 \
|
|
"$(printf '194 Temperature_Internal 0x0022 100 100 000 Old_age Always - 46\n' | parse_disk_temperature)"
|
|
check "SAS Current Drive Temperature" 46 \
|
|
"$(printf 'Current Drive Temperature: 46 C\n' | parse_disk_temperature)"
|
|
check "Temperature_Difference ignored" "" \
|
|
"$(printf '190 Temperature_Difference_from_100 0x0022 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
|
|
check "no temperature reported" "" "$(printf 'Device Model: whatever\n' | parse_disk_temperature)"
|
|
|
|
echo "limit parsing:"
|
|
check "SATA Min/Max limit" 70 "$(printf 'Min/Max Temperature Limit: 0/70 Celsius\n' | parse_disk_limit)"
|
|
check "SATA negative min" 85 "$(printf 'Min/Max Temperature Limit: -41/85 Celsius\n' | parse_disk_limit)"
|
|
check "SAS trip temperature" 60 "$(printf 'Drive Trip Temperature: 60 C\n' | parse_disk_limit)"
|
|
|
|
echo "fan curve (limit 70 -> ramp 52..62, limit 60 -> ramp 42..52):"
|
|
check "SSD at 47c is below its ramp" 18 "$(calculate_interpolated_fan_speed 47 52 62 18 50)"
|
|
check "SSD at 57c is mid ramp" 34 "$(calculate_interpolated_fan_speed 57 52 62 18 50)"
|
|
check "SSD at 62c is flat out" 50 "$(calculate_interpolated_fan_speed 62 52 62 18 50)"
|
|
check "HDD at 47c is mid ramp" 34 "$(calculate_interpolated_fan_speed 47 42 52 18 50)"
|
|
|
|
echo "trend detection:"
|
|
TREND_SAMPLES=5
|
|
: > "$TREND_FILE"
|
|
for t in 40 40 41 40 40; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
|
|
check "flat series does not alarm" 0 "$(rise_over_window 4)"
|
|
: > "$TREND_FILE"
|
|
for t in 38 41 44 47 50; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
|
|
check "rising series reports climb" 12 "$(rise_over_window 4)"
|
|
: > "$TREND_FILE"
|
|
echo "0,50,0,38,20" >> "$TREND_FILE"
|
|
check "partial window stays silent" 0 "$(rise_over_window 4)"
|
|
|
|
echo "fan speed deadband (hold unless the change is worth making):"
|
|
HIGH_FAN_SPEED=50
|
|
check "1% drift holds" 20 "$(next_fan_speed 21 20)"
|
|
check "3% drift holds" 20 "$(next_fan_speed 23 20)"
|
|
check "4% drift holds" 32 "$(next_fan_speed 36 32)"
|
|
check "5% rise is adopted" 25 "$(next_fan_speed 25 20)"
|
|
check "5% fall is adopted" 15 "$(next_fan_speed 15 20)"
|
|
check "full speed always adopted" 50 "$(next_fan_speed 50 20)"
|
|
check "first pass adopts the request" 21 "$(next_fan_speed 21 '')"
|
|
# A re-assert must re-push what is held, not adopt the current want - that
|
|
# leak is what let the setpoint drift 32 -> 36 -> 31 on iz-pve0.
|
|
check "re-assert re-pushes held value" 32 "$(next_fan_speed 34 32)"
|
|
|
|
echo "trend verdict (load is not a fault - only cooling that cannot keep up is):"
|
|
TREND_SAMPLES=5
|
|
HIGH_FAN_SPEED=50
|
|
mkwindow() { : > "$TREND_FILE"; for pair in $@; do echo "0,${pair%%:*},0,${pair%%:*},${pair##*:}" >> "$TREND_FILE"; done; }
|
|
mkwindow 50:20 50:20 51:20 50:20 50:20
|
|
check "flat temps, fans steady" quiet "$(trend_verdict 2 5)"
|
|
mkwindow 50:50 54:50 57:50 59:50 60:50
|
|
check "climbing with fans flat out" no_headroom "$(trend_verdict 2 5)"
|
|
mkwindow 50:20 53:25 56:30 59:35 62:40
|
|
check "climbing as fast as the fans ramp" not_converging "$(trend_verdict 2 5)"
|
|
mkwindow 50:20 55:30 58:35 59:35 59:35
|
|
check "load step that plateaued" quiet "$(trend_verdict 2 5)"
|
|
mkwindow 50:20 54:20 57:20 59:20 60:20
|
|
check "climbing but fans never moved" quiet "$(trend_verdict 2 5)"
|
|
: > "$TREND_FILE"; echo "0,50,0,50,20" >> "$TREND_FILE"
|
|
check "partial window stays silent" quiet "$(trend_verdict 2 5)"
|
|
|
|
echo "ramp/alarm ordering (full cooling must arrive no later than the alarm):"
|
|
ramp_ordering_ok 8 5 && check "full at limit-8, alarm at limit-5" ok ok || check "full at limit-8, alarm at limit-5" ok bad
|
|
ramp_ordering_ok 5 5 && check "full and alarm coincide" ok ok || check "full and alarm coincide" ok bad
|
|
ramp_ordering_ok 2 5 && check "full at limit-2 is rejected" bad ok || check "full at limit-2 is rejected" bad bad
|
|
|
|
echo "alert rate limiting:"
|
|
local out
|
|
out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' )
|
|
check "second alert inside cooldown suppressed" 1 "$out"
|
|
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
|
|
check "recovery notice sent once" 1 "$out"
|
|
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
|
|
check "recovery not repeated" 0 "$out"
|
|
|
|
rm -rf "$tmp"
|
|
[ $fail -eq 0 ] && echo "selftest OK" || { echo "selftest FAILED"; return 1; }
|
|
}
|
|
|
|
if [ "$1" = "selftest" ]; then selftest; exit $?; fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
|
|
if [[ $IDRAC_HOST == "local" ]]; then
|
|
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
|
|
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
|
|
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
|
|
exit 1
|
|
fi
|
|
IDRAC_LOGIN_STRING='open'
|
|
else
|
|
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
|
|
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# This script ONLY runs on Dell Servers
|
|
get_Dell_server_model
|
|
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
|
|
echo "/!\ Your server isn't a Dell product. Exiting." >&2
|
|
exit 1
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Prepare, format and define initial variables
|
|
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
|
|
if [[ $LOW_FAN_SPEED == 0x* ]]; then
|
|
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
|
|
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
|
else
|
|
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
|
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Derive the CPU ramp from what iDRAC says this CPU can actually take.
|
|
CPU_LIMIT=$(cpu_upper_non_critical)
|
|
[ -n "$CPU_LIMIT" ] || CPU_LIMIT=$CPU_LIMIT_FALLBACK
|
|
LOW_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_LOW_OFFSET))
|
|
HIGH_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_HIGH_OFFSET))
|
|
echo "CPU limit ${CPU_LIMIT}c -> fan ramp ${LOW_TEMPERATURE_THRESHOLD}c..${HIGH_TEMPERATURE_THRESHOLD}c"
|
|
#######################################
|
|
|
|
#######################################
|
|
# Determine where to get CPU temps from iDRAC
|
|
# If server model is Gen 14 (*40) or newer
|
|
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
|
|
DELL_POWEREDGE_GEN_14_OR_NEWER=true
|
|
CPU1_TEMPERATURE_INDEX=2
|
|
CPU2_TEMPERATURE_INDEX=4
|
|
else
|
|
DELL_POWEREDGE_GEN_14_OR_NEWER=false
|
|
CPU1_TEMPERATURE_INDEX=1
|
|
CPU2_TEMPERATURE_INDEX=2
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Check if sensors are present on the server
|
|
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
|
|
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
|
|
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
|
if [ -z "$EXHAUST_TEMPERATURE" ]; then
|
|
echo "No exhaust temperature sensor detected."
|
|
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
|
|
fi
|
|
if [ -z "$CPU2_TEMPERATURE" ]; then
|
|
echo "No CPU2 temperature sensor detected."
|
|
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Read every drive's own temperature limit once - they do not change, and
|
|
# smartctl -x is far heavier than the -A used on the fast loop.
|
|
check_ramp_ordering
|
|
cache_disk_limits
|
|
echo "Monitoring $DISK_COUNT_EXPECTED disk(s)."
|
|
LAST_SMART_CHECK=0
|
|
APPLIED_FAN_SPEED=""
|
|
APPLIED_FAN_SPEED_AT=0
|
|
#######################################
|
|
|
|
#######################################
|
|
# One measurement pass: read everything, decide a fan speed, raise alarms.
|
|
one_pass() {
|
|
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
|
retrieve_gpu_temperature
|
|
retrieve_disk_temperatures
|
|
|
|
# Get highest CPU Temp and check if any of them are over heating
|
|
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
|
|
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
|
if [ "$CPU2_TEMPERATURE" -gt "$CPU1_TEMPERATURE" ]; then
|
|
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
|
|
fi
|
|
fi
|
|
|
|
if [ "$HIGHEST_CPU_TEMPERATURE" -gt "$CPU_TEMPERATURE_THRESHOLD" ]; then
|
|
# CPU is overheating - hand cooling back to Dell and say so out loud.
|
|
apply_Dell_fan_control_profile
|
|
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
|
|
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
|
|
raise_alert cpu "Fan control disengaged on $(hostname)" \
|
|
"CPU at ${HIGHEST_CPU_TEMPERATURE}c exceeds the ${CPU_TEMPERATURE_THRESHOLD}c threshold. Dell's default profile has been restored."
|
|
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" 0
|
|
return
|
|
fi
|
|
|
|
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=false
|
|
clear_alert cpu "Fan control restored on $(hostname)"
|
|
|
|
# CPU and GPU share one curve; disks are interpolated against their own limits
|
|
# in retrieve_disk_temperatures. Each source asks for a speed, loudest wins -
|
|
# temperatures from different classes of hardware are not comparable directly.
|
|
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_CPU_TEMPERATURE" \
|
|
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
|
# The GPU has its own limit and so gets its own ramp, rather than being
|
|
# compared against the CPU's on a shared scale.
|
|
if [ "$GPU_TEMPERATURE" -gt 0 ]; then
|
|
local gpu_speed
|
|
gpu_speed=$(calculate_interpolated_fan_speed "$GPU_TEMPERATURE" \
|
|
$((GPU_TEMPERATURE_THRESHOLD - CPU_RAMP_LOW_OFFSET)) $GPU_TEMPERATURE_THRESHOLD \
|
|
$LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
|
[ "$gpu_speed" -gt "$CPU_GPU_FAN_SPEED" ] && CPU_GPU_FAN_SPEED=$gpu_speed
|
|
fi
|
|
|
|
DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED
|
|
FAN_SPEED_DRIVER="cpu/gpu"
|
|
if [ "$DISK_FAN_SPEED" -gt "$DECIMAL_CURRENT_FAN_SPEED" ]; then
|
|
DECIMAL_CURRENT_FAN_SPEED=$DISK_FAN_SPEED
|
|
FAN_SPEED_DRIVER="disk"
|
|
fi
|
|
|
|
# Hold the current speed unless the change is worth making - see the deadband
|
|
# note in monitor.sh. Chasing every degree is what made this louder than stock.
|
|
local target now held=""
|
|
target=$(next_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED")
|
|
[ "$target" = "$DECIMAL_CURRENT_FAN_SPEED" ] || held=" held, want $DECIMAL_CURRENT_FAN_SPEED"
|
|
now=$(date +%s)
|
|
if [ "$target" != "$APPLIED_FAN_SPEED" ] ||
|
|
[ $((now - APPLIED_FAN_SPEED_AT)) -ge "$FAN_REASSERT_INTERVAL" ]; then
|
|
apply_user_fan_control "$target"
|
|
APPLIED_FAN_SPEED=$target
|
|
APPLIED_FAN_SPEED_AT=$now
|
|
fi
|
|
|
|
COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)"
|
|
|
|
# Record what the fans are actually doing, not what was merely requested.
|
|
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$APPLIED_FAN_SPEED"
|
|
check_alarms
|
|
}
|
|
|
|
check_alarms() {
|
|
# A drive within DISK_ALARM_OFFSET of its own limit: fans are already flat out
|
|
# for it and it is still climbing.
|
|
if [ "$HOTTEST_DISK_TEMPERATURE" -gt 0 ] &&
|
|
[ "$HOTTEST_DISK_TEMPERATURE" -ge $((HOTTEST_DISK_LIMIT - DISK_ALARM_OFFSET)) ]; then
|
|
raise_alert disk_temp "Disk over temperature on $(hostname)" \
|
|
"$HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c, its own limit is ${HOTTEST_DISK_LIMIT}c. Fans at ${DECIMAL_CURRENT_FAN_SPEED}%."
|
|
else
|
|
clear_alert disk_temp "Disk temperature normal on $(hostname)"
|
|
fi
|
|
|
|
if [ "$GPU_TEMPERATURE" -gt "$GPU_TEMPERATURE_THRESHOLD" ]; then
|
|
raise_alert gpu "GPU over temperature on $(hostname)" \
|
|
"GPU at ${GPU_TEMPERATURE}c exceeds the ${GPU_TEMPERATURE_THRESHOLD}c threshold."
|
|
else
|
|
clear_alert gpu "GPU temperature normal on $(hostname)"
|
|
fi
|
|
|
|
# Climbing is only a fault if the fans are not answering it. A busy machine
|
|
# heats up and plateaus; a dying fan or blocked intake keeps climbing. See
|
|
# trend_verdict() - this stays quiet through hours of legitimate load.
|
|
local cpu_verdict disk_verdict cpu_rise disk_rise minutes
|
|
cpu_verdict=$(trend_verdict 2 5)
|
|
disk_verdict=$(trend_verdict 4 5)
|
|
cpu_rise=$(rise_over_window 2)
|
|
disk_rise=$(rise_over_window 4)
|
|
minutes=$((TREND_SAMPLES * CHECK_INTERVAL / 60))
|
|
if [ "$cpu_verdict" = no_headroom ] || [ "$disk_verdict" = no_headroom ]; then
|
|
raise_alert trend "Out of cooling headroom on $(hostname)" \
|
|
"Fans are at ${HIGH_FAN_SPEED}% and temperatures are still climbing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c ($HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c of ${HOTTEST_DISK_LIMIT}c). There is no cooling left to apply."
|
|
elif [ "$cpu_verdict" = not_converging ] || [ "$disk_verdict" = not_converging ]; then
|
|
raise_alert trend "Temperature outrunning the fans on $(hostname)" \
|
|
"Temperatures are climbing and the fans are ramping with them, but the rise is not slowing. Over the last ${minutes} minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c, fans now ${APPLIED_FAN_SPEED}%. Check airflow, intake and fan health."
|
|
else
|
|
clear_alert trend "Temperature stabilised on $(hostname)"
|
|
fi
|
|
|
|
# A drive that stops answering is either asleep or gone. Worth knowing which.
|
|
if [ "$DISKS_READ" -lt "$DISK_COUNT_EXPECTED" ]; then
|
|
raise_alert disk_count "Disk missing on $(hostname)" \
|
|
"Read $DISKS_READ of $DISK_COUNT_EXPECTED disks. A drive is in standby, has dropped off the bus, or has failed."
|
|
else
|
|
clear_alert disk_count "All disks reporting on $(hostname)"
|
|
fi
|
|
|
|
# SMART is expensive across two dozen drives - hourly, not every loop.
|
|
local now; now=$(date +%s)
|
|
if [ $((now - LAST_SMART_CHECK)) -ge "$SMART_CHECK_INTERVAL" ]; then
|
|
LAST_SMART_CHECK=$now
|
|
check_disk_health
|
|
fi
|
|
}
|
|
#######################################
|
|
|
|
case "${1:-run}" in
|
|
once)
|
|
one_pass
|
|
echo "$COMMENT"
|
|
log_line "$COMMENT"
|
|
;;
|
|
disks)
|
|
for dev in $(disk_devices); do
|
|
printf '%-12s %-4s limit=%sc\n' "$dev" "$(disk_temperature "$dev")c" "$(disk_limit "$dev")"
|
|
done
|
|
;;
|
|
run)
|
|
while true; do
|
|
sleep $CHECK_INTERVAL &
|
|
SLEEP_PROCESS_PID=$!
|
|
one_pass
|
|
log_line "$COMMENT"
|
|
wait $SLEEP_PROCESS_PID
|
|
done
|
|
;;
|
|
*)
|
|
echo "usage: $0 [run|once|disks|selftest]" >&2
|
|
exit 1
|
|
;;
|
|
esac
|