Add disk temperature monitoring, trend tracking and email alarms
Fan speed now takes the loudest request from CPU/GPU and from each drive interpolated against its own reported temperature limit, so bay position and drive technology stop mattering. Reported limits are clamped by class because WD Reds report the SCT critical limit (85) rather than an operating maximum. Adds a rolling temperature history and four alarms (absolute, rising trend, SMART health, disk count), rate limited per key with a one hour cooldown, delivered by mail through the host relay. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
314
fan_speed.sh
Executable file
314
fan_speed.sh
Executable file
@@ -0,0 +1,314 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Dell PowerEdge fan control driven by CPU, GPU and disk temperature, with
|
||||
# trend tracking and email alarms.
|
||||
#
|
||||
# Usage: fan_speed.sh [run|once|disks|selftest] (default: run)
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
source "$SCRIPT_DIR/functions.sh"
|
||||
source "$SCRIPT_DIR/monitor.sh"
|
||||
|
||||
# Trap the signals for container exit and run graceful_exit function
|
||||
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
|
||||
|
||||
# Variable defintions
|
||||
# LOW_FAN_SPEED = Lowest fan speed
|
||||
# HIGH_FAN_SPEED = max fan speed. (0-100)
|
||||
# IDRAC_HOST = local or IP of iDRAC
|
||||
# IDRAC_USERNAME = username of iDRAC
|
||||
# IDRAC_PASSWORD = password if iDRAC
|
||||
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
|
||||
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
|
||||
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
|
||||
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
|
||||
|
||||
IDRAC_HOST=local
|
||||
IDRAC_USERNAME=root
|
||||
IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin}
|
||||
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
|
||||
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
|
||||
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
|
||||
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
|
||||
LOW_FAN_SPEED=18 #only decimal numbers
|
||||
HIGH_FAN_SPEED=50 #only decimal numbers
|
||||
TABLE_HEADER_PRINT_INTERVAL=2
|
||||
|
||||
# Disk thresholds, alarm routing and trend window all live in monitor.sh.
|
||||
|
||||
#######################################
|
||||
# Self test - pure logic only, no IPMI and no drives touched.
|
||||
selftest() {
|
||||
local tmp; tmp=$(mktemp -d)
|
||||
export STATE_DIR="$tmp/state" TREND_FILE="$tmp/temps.csv" LOG_FILE="$tmp/log" DRY_RUN=1
|
||||
local fail=0
|
||||
check() { # check <description> <expected> <actual>
|
||||
if [ "$2" = "$3" ]; then echo " ok $1"; else echo " FAIL $1: expected '$2' got '$3'"; fail=1; fi
|
||||
}
|
||||
|
||||
echo "temperature parsing:"
|
||||
check "SATA Temperature_Celsius" 35 \
|
||||
"$(printf '194 Temperature_Celsius 0x0022 115 099 000 Old_age Always - 35\n' | parse_disk_temperature)"
|
||||
check "Seagate Airflow_Temperature_Cel" 41 \
|
||||
"$(printf '190 Airflow_Temperature_Cel 0x0032 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
|
||||
check "Intel Temperature_Internal" 46 \
|
||||
"$(printf '194 Temperature_Internal 0x0022 100 100 000 Old_age Always - 46\n' | parse_disk_temperature)"
|
||||
check "SAS Current Drive Temperature" 46 \
|
||||
"$(printf 'Current Drive Temperature: 46 C\n' | parse_disk_temperature)"
|
||||
check "Temperature_Difference ignored" "" \
|
||||
"$(printf '190 Temperature_Difference_from_100 0x0022 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
|
||||
check "no temperature reported" "" "$(printf 'Device Model: whatever\n' | parse_disk_temperature)"
|
||||
|
||||
echo "limit parsing:"
|
||||
check "SATA Min/Max limit" 70 "$(printf 'Min/Max Temperature Limit: 0/70 Celsius\n' | parse_disk_limit)"
|
||||
check "SATA negative min" 85 "$(printf 'Min/Max Temperature Limit: -41/85 Celsius\n' | parse_disk_limit)"
|
||||
check "SAS trip temperature" 60 "$(printf 'Drive Trip Temperature: 60 C\n' | parse_disk_limit)"
|
||||
|
||||
echo "fan curve (limit 70 -> ramp 52..62, limit 60 -> ramp 42..52):"
|
||||
check "SSD at 47c is below its ramp" 18 "$(calculate_interpolated_fan_speed 47 52 62 18 50)"
|
||||
check "SSD at 57c is mid ramp" 34 "$(calculate_interpolated_fan_speed 57 52 62 18 50)"
|
||||
check "SSD at 62c is flat out" 50 "$(calculate_interpolated_fan_speed 62 52 62 18 50)"
|
||||
check "HDD at 47c is mid ramp" 34 "$(calculate_interpolated_fan_speed 47 42 52 18 50)"
|
||||
|
||||
echo "trend detection:"
|
||||
TREND_SAMPLES=5
|
||||
: > "$TREND_FILE"
|
||||
for t in 40 40 41 40 40; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
|
||||
check "flat series does not alarm" 0 "$(rise_over_window 4)"
|
||||
: > "$TREND_FILE"
|
||||
for t in 38 41 44 47 50; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
|
||||
check "rising series reports climb" 12 "$(rise_over_window 4)"
|
||||
: > "$TREND_FILE"
|
||||
echo "0,50,0,38,20" >> "$TREND_FILE"
|
||||
check "partial window stays silent" 0 "$(rise_over_window 4)"
|
||||
|
||||
echo "alert rate limiting:"
|
||||
local out
|
||||
out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' )
|
||||
check "second alert inside cooldown suppressed" 1 "$out"
|
||||
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
|
||||
check "recovery notice sent once" 1 "$out"
|
||||
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
|
||||
check "recovery not repeated" 0 "$out"
|
||||
|
||||
rm -rf "$tmp"
|
||||
[ $fail -eq 0 ] && echo "selftest OK" || { echo "selftest FAILED"; return 1; }
|
||||
}
|
||||
|
||||
if [ "$1" = "selftest" ]; then selftest; exit $?; fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
|
||||
if [[ $IDRAC_HOST == "local" ]]; then
|
||||
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
|
||||
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
|
||||
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
IDRAC_LOGIN_STRING='open'
|
||||
else
|
||||
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
|
||||
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# This script ONLY runs on Dell Servers
|
||||
get_Dell_server_model
|
||||
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
|
||||
echo "/!\ Your server isn't a Dell product. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Prepare, format and define initial variables
|
||||
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
|
||||
if [[ $LOW_FAN_SPEED == 0x* ]]; then
|
||||
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
|
||||
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
||||
else
|
||||
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
||||
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# This returns the lowest temp between CPU and GPU highest thresholds
|
||||
# Used to calcuate fan speed interpolation
|
||||
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
|
||||
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
|
||||
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
|
||||
else
|
||||
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Determine where to get CPU temps from iDRAC
|
||||
# If server model is Gen 14 (*40) or newer
|
||||
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
|
||||
DELL_POWEREDGE_GEN_14_OR_NEWER=true
|
||||
CPU1_TEMPERATURE_INDEX=2
|
||||
CPU2_TEMPERATURE_INDEX=4
|
||||
else
|
||||
DELL_POWEREDGE_GEN_14_OR_NEWER=false
|
||||
CPU1_TEMPERATURE_INDEX=1
|
||||
CPU2_TEMPERATURE_INDEX=2
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Check if sensors are present on the server
|
||||
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
|
||||
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
|
||||
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
if [ -z "$EXHAUST_TEMPERATURE" ]; then
|
||||
echo "No exhaust temperature sensor detected."
|
||||
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
|
||||
fi
|
||||
if [ -z "$CPU2_TEMPERATURE" ]; then
|
||||
echo "No CPU2 temperature sensor detected."
|
||||
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Read every drive's own temperature limit once - they do not change, and
|
||||
# smartctl -x is far heavier than the -A used on the fast loop.
|
||||
cache_disk_limits
|
||||
echo "Monitoring $DISK_COUNT_EXPECTED disk(s)."
|
||||
LAST_SMART_CHECK=0
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# One measurement pass: read everything, decide a fan speed, raise alarms.
|
||||
one_pass() {
|
||||
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
retrieve_gpu_temperature
|
||||
retrieve_disk_temperatures
|
||||
|
||||
# Get highest CPU Temp and check if any of them are over heating
|
||||
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
|
||||
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
||||
if [ "$CPU2_TEMPERATURE" -gt "$CPU1_TEMPERATURE" ]; then
|
||||
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$HIGHEST_CPU_TEMPERATURE" -gt "$CPU_TEMPERATURE_THRESHOLD" ]; then
|
||||
# CPU is overheating - hand cooling back to Dell and say so out loud.
|
||||
apply_Dell_fan_control_profile
|
||||
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
|
||||
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
|
||||
raise_alert cpu "Fan control disengaged on $(hostname)" \
|
||||
"CPU at ${HIGHEST_CPU_TEMPERATURE}c exceeds the ${CPU_TEMPERATURE_THRESHOLD}c threshold. Dell's default profile has been restored."
|
||||
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" 0
|
||||
return
|
||||
fi
|
||||
|
||||
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=false
|
||||
clear_alert cpu "Fan control restored on $(hostname)"
|
||||
|
||||
# CPU and GPU share one curve; disks are interpolated against their own limits
|
||||
# in retrieve_disk_temperatures. Each source asks for a speed, loudest wins -
|
||||
# temperatures from different classes of hardware are not comparable directly.
|
||||
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
|
||||
if [ "$GPU_TEMPERATURE" -gt "$HIGHEST_CPU_TEMPERATURE" ]; then
|
||||
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
|
||||
fi
|
||||
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
|
||||
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_TEMPERATURE" \
|
||||
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
||||
else
|
||||
CPU_GPU_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
|
||||
fi
|
||||
|
||||
DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED
|
||||
FAN_SPEED_DRIVER="cpu/gpu"
|
||||
if [ "$DISK_FAN_SPEED" -gt "$DECIMAL_CURRENT_FAN_SPEED" ]; then
|
||||
DECIMAL_CURRENT_FAN_SPEED=$DISK_FAN_SPEED
|
||||
FAN_SPEED_DRIVER="disk"
|
||||
fi
|
||||
|
||||
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
|
||||
|
||||
COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED($FAN_SPEED_DRIVER)"
|
||||
|
||||
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$DECIMAL_CURRENT_FAN_SPEED"
|
||||
check_alarms
|
||||
}
|
||||
|
||||
check_alarms() {
|
||||
# A drive within DISK_ALARM_OFFSET of its own limit: fans are already flat out
|
||||
# for it and it is still climbing.
|
||||
if [ "$HOTTEST_DISK_TEMPERATURE" -gt 0 ] &&
|
||||
[ "$HOTTEST_DISK_TEMPERATURE" -ge $((HOTTEST_DISK_LIMIT - DISK_ALARM_OFFSET)) ]; then
|
||||
raise_alert disk_temp "Disk over temperature on $(hostname)" \
|
||||
"$HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c, its own limit is ${HOTTEST_DISK_LIMIT}c. Fans at ${DECIMAL_CURRENT_FAN_SPEED}%."
|
||||
else
|
||||
clear_alert disk_temp "Disk temperature normal on $(hostname)"
|
||||
fi
|
||||
|
||||
if [ "$GPU_TEMPERATURE" -gt "$GPU_TEMPERATURE_THRESHOLD" ]; then
|
||||
raise_alert gpu "GPU over temperature on $(hostname)" \
|
||||
"GPU at ${GPU_TEMPERATURE}c exceeds the ${GPU_TEMPERATURE_THRESHOLD}c threshold."
|
||||
else
|
||||
clear_alert gpu "GPU temperature normal on $(hostname)"
|
||||
fi
|
||||
|
||||
# Climbing steadily while still under every threshold - a dying fan or a
|
||||
# blocked intake looks exactly like this long before anything crosses a limit.
|
||||
local cpu_rise disk_rise
|
||||
cpu_rise=$(rise_over_window 2)
|
||||
disk_rise=$(rise_over_window 4)
|
||||
if [ "$cpu_rise" -ge "$TREND_RISE_ALARM" ] || [ "$disk_rise" -ge "$TREND_RISE_ALARM" ]; then
|
||||
raise_alert trend "Temperature climbing on $(hostname)" \
|
||||
"Over the last $((TREND_SAMPLES * CHECK_INTERVAL / 60)) minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c. Nothing has crossed a threshold yet. Check airflow and fans."
|
||||
else
|
||||
clear_alert trend "Temperature stabilised on $(hostname)"
|
||||
fi
|
||||
|
||||
# A drive that stops answering is either asleep or gone. Worth knowing which.
|
||||
if [ "$DISKS_READ" -lt "$DISK_COUNT_EXPECTED" ]; then
|
||||
raise_alert disk_count "Disk missing on $(hostname)" \
|
||||
"Read $DISKS_READ of $DISK_COUNT_EXPECTED disks. A drive is in standby, has dropped off the bus, or has failed."
|
||||
else
|
||||
clear_alert disk_count "All disks reporting on $(hostname)"
|
||||
fi
|
||||
|
||||
# SMART is expensive across two dozen drives - hourly, not every loop.
|
||||
local now; now=$(date +%s)
|
||||
if [ $((now - LAST_SMART_CHECK)) -ge "$SMART_CHECK_INTERVAL" ]; then
|
||||
LAST_SMART_CHECK=$now
|
||||
check_disk_health
|
||||
fi
|
||||
}
|
||||
#######################################
|
||||
|
||||
case "${1:-run}" in
|
||||
once)
|
||||
one_pass
|
||||
echo "$COMMENT"
|
||||
log_line "$COMMENT"
|
||||
;;
|
||||
disks)
|
||||
for dev in $(disk_devices); do
|
||||
printf '%-12s %-4s limit=%sc\n' "$dev" "$(disk_temperature "$dev")c" "$(disk_limit "$dev")"
|
||||
done
|
||||
;;
|
||||
run)
|
||||
while true; do
|
||||
sleep $CHECK_INTERVAL &
|
||||
SLEEP_PROCESS_PID=$!
|
||||
one_pass
|
||||
log_line "$COMMENT"
|
||||
wait $SLEEP_PROCESS_PID
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 [run|once|disks|selftest]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user