izebra_projects/fan_speed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
562d8e8d466314aaa4337aae85a8b09077c3bd96
fan_speed/fan_speed.sh
Zeb Hering 562d8e8d46 Add disk temperature monitoring, trend tracking and email alarms 
Fan speed now takes the loudest request from CPU/GPU and from each drive
interpolated against its own reported temperature limit, so bay position
and drive technology stop mattering. Reported limits are clamped by class
because WD Reds report the SCT critical limit (85) rather than an
operating maximum.

Adds a rolling temperature history and four alarms (absolute, rising
trend, SMART health, disk count), rate limited per key with a one hour
cooldown, delivered by mail through the host relay.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-28 18:36:07 -07:00

315 lines
13 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Dell PowerEdge fan control driven by CPU, GPU and disk temperature, with
# trend tracking and email alarms.
#
# Usage: fan_speed.sh [run|once|disks|selftest] (default: run)
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source "$SCRIPT_DIR/functions.sh"
source "$SCRIPT_DIR/monitor.sh"
# Trap the signals for container exit and run graceful_exit function
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
# Variable defintions
# LOW_FAN_SPEED = Lowest fan speed
# HIGH_FAN_SPEED = max fan speed. (0-100)
# IDRAC_HOST = local or IP of iDRAC
# IDRAC_USERNAME = username of iDRAC
# IDRAC_PASSWORD = password if iDRAC
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
IDRAC_HOST=local
IDRAC_USERNAME=root
IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin}
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
LOW_FAN_SPEED=18 #only decimal numbers
HIGH_FAN_SPEED=50 #only decimal numbers
TABLE_HEADER_PRINT_INTERVAL=2
# Disk thresholds, alarm routing and trend window all live in monitor.sh.
#######################################
# Self test - pure logic only, no IPMI and no drives touched.
selftest() {
local tmp; tmp=$(mktemp -d)
export STATE_DIR="$tmp/state" TREND_FILE="$tmp/temps.csv" LOG_FILE="$tmp/log" DRY_RUN=1
local fail=0
check() { # check <description> <expected> <actual>
if [ "$2" = "$3" ]; then echo " ok $1"; else echo " FAIL $1: expected '$2' got '$3'"; fail=1; fi
}
echo "temperature parsing:"
check "SATA Temperature_Celsius" 35 \
"$(printf '194 Temperature_Celsius 0x0022 115 099 000 Old_age Always - 35\n' | parse_disk_temperature)"
check "Seagate Airflow_Temperature_Cel" 41 \
"$(printf '190 Airflow_Temperature_Cel 0x0032 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
check "Intel Temperature_Internal" 46 \
"$(printf '194 Temperature_Internal 0x0022 100 100 000 Old_age Always - 46\n' | parse_disk_temperature)"
check "SAS Current Drive Temperature" 46 \
"$(printf 'Current Drive Temperature: 46 C\n' | parse_disk_temperature)"
check "Temperature_Difference ignored" "" \
"$(printf '190 Temperature_Difference_from_100 0x0022 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
check "no temperature reported" "" "$(printf 'Device Model: whatever\n' | parse_disk_temperature)"
echo "limit parsing:"
check "SATA Min/Max limit" 70 "$(printf 'Min/Max Temperature Limit: 0/70 Celsius\n' | parse_disk_limit)"
check "SATA negative min" 85 "$(printf 'Min/Max Temperature Limit: -41/85 Celsius\n' | parse_disk_limit)"
check "SAS trip temperature" 60 "$(printf 'Drive Trip Temperature: 60 C\n' | parse_disk_limit)"
echo "fan curve (limit 70 -> ramp 52..62, limit 60 -> ramp 42..52):"
check "SSD at 47c is below its ramp" 18 "$(calculate_interpolated_fan_speed 47 52 62 18 50)"
check "SSD at 57c is mid ramp" 34 "$(calculate_interpolated_fan_speed 57 52 62 18 50)"
check "SSD at 62c is flat out" 50 "$(calculate_interpolated_fan_speed 62 52 62 18 50)"
check "HDD at 47c is mid ramp" 34 "$(calculate_interpolated_fan_speed 47 42 52 18 50)"
echo "trend detection:"
TREND_SAMPLES=5
: > "$TREND_FILE"
for t in 40 40 41 40 40; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
check "flat series does not alarm" 0 "$(rise_over_window 4)"
: > "$TREND_FILE"
for t in 38 41 44 47 50; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
check "rising series reports climb" 12 "$(rise_over_window 4)"
: > "$TREND_FILE"
echo "0,50,0,38,20" >> "$TREND_FILE"
check "partial window stays silent" 0 "$(rise_over_window 4)"
echo "alert rate limiting:"
local out
out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' )
check "second alert inside cooldown suppressed" 1 "$out"
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
check "recovery notice sent once" 1 "$out"
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
check "recovery not repeated" 0 "$out"
rm -rf "$tmp"
[ $fail -eq 0 ] && echo "selftest OK" || { echo "selftest FAILED"; return 1; }
}
if [ "$1" = "selftest" ]; then selftest; exit $?; fi
#######################################
#######################################
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
if [[ $IDRAC_HOST == "local" ]]; then
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
exit 1
fi
IDRAC_LOGIN_STRING='open'
else
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
fi
#######################################
#######################################
# This script ONLY runs on Dell Servers
get_Dell_server_model
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
echo "/!\ Your server isn't a Dell product. Exiting." >&2
exit 1
fi
#######################################
#######################################
# Prepare, format and define initial variables
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
if [[ $LOW_FAN_SPEED == 0x* ]]; then
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
else
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
fi
#######################################
#######################################
# This returns the lowest temp between CPU and GPU highest thresholds
# Used to calcuate fan speed interpolation
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
else
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
fi
#######################################
#######################################
# Determine where to get CPU temps from iDRAC
# If server model is Gen 14 (*40) or newer
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
DELL_POWEREDGE_GEN_14_OR_NEWER=true
CPU1_TEMPERATURE_INDEX=2
CPU2_TEMPERATURE_INDEX=4
else
DELL_POWEREDGE_GEN_14_OR_NEWER=false
CPU1_TEMPERATURE_INDEX=1
CPU2_TEMPERATURE_INDEX=2
fi
#######################################
#######################################
# Check if sensors are present on the server
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
if [ -z "$EXHAUST_TEMPERATURE" ]; then
echo "No exhaust temperature sensor detected."
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
fi
if [ -z "$CPU2_TEMPERATURE" ]; then
echo "No CPU2 temperature sensor detected."
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
fi
#######################################
#######################################
# Read every drive's own temperature limit once - they do not change, and
# smartctl -x is far heavier than the -A used on the fast loop.
cache_disk_limits
echo "Monitoring $DISK_COUNT_EXPECTED disk(s)."
LAST_SMART_CHECK=0
#######################################
#######################################
# One measurement pass: read everything, decide a fan speed, raise alarms.
one_pass() {
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
retrieve_gpu_temperature
retrieve_disk_temperatures
# Get highest CPU Temp and check if any of them are over heating
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
if [ "$CPU2_TEMPERATURE" -gt "$CPU1_TEMPERATURE" ]; then
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
fi
fi
if [ "$HIGHEST_CPU_TEMPERATURE" -gt "$CPU_TEMPERATURE_THRESHOLD" ]; then
# CPU is overheating - hand cooling back to Dell and say so out loud.
apply_Dell_fan_control_profile
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
raise_alert cpu "Fan control disengaged on $(hostname)" \
"CPU at ${HIGHEST_CPU_TEMPERATURE}c exceeds the ${CPU_TEMPERATURE_THRESHOLD}c threshold. Dell's default profile has been restored."
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" 0
return
fi
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=false
clear_alert cpu "Fan control restored on $(hostname)"
# CPU and GPU share one curve; disks are interpolated against their own limits
# in retrieve_disk_temperatures. Each source asks for a speed, loudest wins -
# temperatures from different classes of hardware are not comparable directly.
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
if [ "$GPU_TEMPERATURE" -gt "$HIGHEST_CPU_TEMPERATURE" ]; then
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
fi
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_TEMPERATURE" \
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
else
CPU_GPU_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
fi
DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED
FAN_SPEED_DRIVER="cpu/gpu"
if [ "$DISK_FAN_SPEED" -gt "$DECIMAL_CURRENT_FAN_SPEED" ]; then
DECIMAL_CURRENT_FAN_SPEED=$DISK_FAN_SPEED
FAN_SPEED_DRIVER="disk"
fi
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED($FAN_SPEED_DRIVER)"
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$DECIMAL_CURRENT_FAN_SPEED"
check_alarms
}
check_alarms() {
# A drive within DISK_ALARM_OFFSET of its own limit: fans are already flat out
# for it and it is still climbing.
if [ "$HOTTEST_DISK_TEMPERATURE" -gt 0 ] &&
[ "$HOTTEST_DISK_TEMPERATURE" -ge $((HOTTEST_DISK_LIMIT - DISK_ALARM_OFFSET)) ]; then
raise_alert disk_temp "Disk over temperature on $(hostname)" \
"$HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c, its own limit is ${HOTTEST_DISK_LIMIT}c. Fans at ${DECIMAL_CURRENT_FAN_SPEED}%."
else
clear_alert disk_temp "Disk temperature normal on $(hostname)"
fi
if [ "$GPU_TEMPERATURE" -gt "$GPU_TEMPERATURE_THRESHOLD" ]; then
raise_alert gpu "GPU over temperature on $(hostname)" \
"GPU at ${GPU_TEMPERATURE}c exceeds the ${GPU_TEMPERATURE_THRESHOLD}c threshold."
else
clear_alert gpu "GPU temperature normal on $(hostname)"
fi
# Climbing steadily while still under every threshold - a dying fan or a
# blocked intake looks exactly like this long before anything crosses a limit.
local cpu_rise disk_rise
cpu_rise=$(rise_over_window 2)
disk_rise=$(rise_over_window 4)
if [ "$cpu_rise" -ge "$TREND_RISE_ALARM" ] || [ "$disk_rise" -ge "$TREND_RISE_ALARM" ]; then
raise_alert trend "Temperature climbing on $(hostname)" \
"Over the last $((TREND_SAMPLES * CHECK_INTERVAL / 60)) minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c. Nothing has crossed a threshold yet. Check airflow and fans."
else
clear_alert trend "Temperature stabilised on $(hostname)"
fi
# A drive that stops answering is either asleep or gone. Worth knowing which.
if [ "$DISKS_READ" -lt "$DISK_COUNT_EXPECTED" ]; then
raise_alert disk_count "Disk missing on $(hostname)" \
"Read $DISKS_READ of $DISK_COUNT_EXPECTED disks. A drive is in standby, has dropped off the bus, or has failed."
else
clear_alert disk_count "All disks reporting on $(hostname)"
fi
# SMART is expensive across two dozen drives - hourly, not every loop.
local now; now=$(date +%s)
if [ $((now - LAST_SMART_CHECK)) -ge "$SMART_CHECK_INTERVAL" ]; then
LAST_SMART_CHECK=$now
check_disk_health
fi
}
#######################################
case "${1:-run}" in
once)
one_pass
echo "$COMMENT"
log_line "$COMMENT"
;;
disks)
for dev in $(disk_devices); do
printf '%-12s %-4s limit=%sc\n' "$dev" "$(disk_temperature "$dev")c" "$(disk_limit "$dev")"
done
;;
run)
while true; do
sleep $CHECK_INTERVAL &
SLEEP_PROCESS_PID=$!
one_pass
log_line "$COMMENT"
wait $SLEEP_PROCESS_PID
done
;;
*)
echo "usage: $0 [run|once|disks|selftest]" >&2
exit 1
;;
esac
Reference in New Issue View Git Blame Copy Permalink