izebra_projects/fan_speed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
56ae7542a2c142850e8a60f46feee97c809e1efe
fan_speed/fan_speed.sh
Zeb Hering 56ae7542a2 Hold fan speed unless the change is worth making 
The setpoint was a continuous function of instantaneous temperature and
was pushed to the BMC every pass, so ~1% of fan per degree of CPU noise
meant the fans never settled. Measured on iz-pve0: stock iDRAC held one
speed for 5 minutes through 57-59c jitter while this script made 7
changes in 16 minutes off the same signal.

Adds a 5% deadband around the applied speed, with full-speed requests
never held back and a periodic re-assert in case the BMC forgets.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-28 20:07:33 -07:00

338 lines
15 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Dell PowerEdge fan control driven by CPU, GPU and disk temperature, with
# trend tracking and email alarms.
#
# Usage: fan_speed.sh [run|once|disks|selftest] (default: run)
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source "$SCRIPT_DIR/functions.sh"
source "$SCRIPT_DIR/monitor.sh"
# Trap the signals for container exit and run graceful_exit function
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
# Variable defintions
# LOW_FAN_SPEED = Lowest fan speed
# HIGH_FAN_SPEED = max fan speed. (0-100)
# IDRAC_HOST = local or IP of iDRAC
# IDRAC_USERNAME = username of iDRAC
# IDRAC_PASSWORD = password if iDRAC
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
IDRAC_HOST=local
IDRAC_USERNAME=root
IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin}
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
LOW_FAN_SPEED=18 #only decimal numbers
HIGH_FAN_SPEED=50 #only decimal numbers
TABLE_HEADER_PRINT_INTERVAL=2
# Disk thresholds, alarm routing and trend window all live in monitor.sh.
#######################################
# Self test - pure logic only, no IPMI and no drives touched.
selftest() {
local tmp; tmp=$(mktemp -d)
export STATE_DIR="$tmp/state" TREND_FILE="$tmp/temps.csv" LOG_FILE="$tmp/log" DRY_RUN=1
local fail=0
check() { # check <description> <expected> <actual>
if [ "$2" = "$3" ]; then echo " ok $1"; else echo " FAIL $1: expected '$2' got '$3'"; fail=1; fi
}
echo "temperature parsing:"
check "SATA Temperature_Celsius" 35 \
"$(printf '194 Temperature_Celsius 0x0022 115 099 000 Old_age Always - 35\n' | parse_disk_temperature)"
check "Seagate Airflow_Temperature_Cel" 41 \
"$(printf '190 Airflow_Temperature_Cel 0x0032 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
check "Intel Temperature_Internal" 46 \
"$(printf '194 Temperature_Internal 0x0022 100 100 000 Old_age Always - 46\n' | parse_disk_temperature)"
check "SAS Current Drive Temperature" 46 \
"$(printf 'Current Drive Temperature: 46 C\n' | parse_disk_temperature)"
check "Temperature_Difference ignored" "" \
"$(printf '190 Temperature_Difference_from_100 0x0022 059 051 000 Old_age Always - 41\n' | parse_disk_temperature)"
check "no temperature reported" "" "$(printf 'Device Model: whatever\n' | parse_disk_temperature)"
echo "limit parsing:"
check "SATA Min/Max limit" 70 "$(printf 'Min/Max Temperature Limit: 0/70 Celsius\n' | parse_disk_limit)"
check "SATA negative min" 85 "$(printf 'Min/Max Temperature Limit: -41/85 Celsius\n' | parse_disk_limit)"
check "SAS trip temperature" 60 "$(printf 'Drive Trip Temperature: 60 C\n' | parse_disk_limit)"
echo "fan curve (limit 70 -> ramp 52..62, limit 60 -> ramp 42..52):"
check "SSD at 47c is below its ramp" 18 "$(calculate_interpolated_fan_speed 47 52 62 18 50)"
check "SSD at 57c is mid ramp" 34 "$(calculate_interpolated_fan_speed 57 52 62 18 50)"
check "SSD at 62c is flat out" 50 "$(calculate_interpolated_fan_speed 62 52 62 18 50)"
check "HDD at 47c is mid ramp" 34 "$(calculate_interpolated_fan_speed 47 42 52 18 50)"
echo "trend detection:"
TREND_SAMPLES=5
: > "$TREND_FILE"
for t in 40 40 41 40 40; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
check "flat series does not alarm" 0 "$(rise_over_window 4)"
: > "$TREND_FILE"
for t in 38 41 44 47 50; do echo "0,50,0,$t,20" >> "$TREND_FILE"; done
check "rising series reports climb" 12 "$(rise_over_window 4)"
: > "$TREND_FILE"
echo "0,50,0,38,20" >> "$TREND_FILE"
check "partial window stays silent" 0 "$(rise_over_window 4)"
echo "fan speed deadband (hold unless the change is worth making):"
local now; now=$(date +%s)
HIGH_FAN_SPEED=50
should_apply_fan_speed 21 20 "$now" && check "1% drift holds" hold apply || check "1% drift holds" hold hold
should_apply_fan_speed 23 20 "$now" && check "3% drift holds" hold apply || check "3% drift holds" hold hold
should_apply_fan_speed 25 20 "$now" && check "5% rise applies" apply apply || check "5% rise applies" apply hold
should_apply_fan_speed 15 20 "$now" && check "5% fall applies" apply apply || check "5% fall applies" apply hold
should_apply_fan_speed 50 20 "$now" && check "full speed always applies" apply apply || check "full speed always applies" apply hold
should_apply_fan_speed 21 "" "$now" && check "first pass always applies" apply apply || check "first pass always applies" apply hold
should_apply_fan_speed 21 20 $((now - 400)) && check "stale setpoint is reasserted" apply apply || check "stale setpoint is reasserted" apply hold
echo "alert rate limiting:"
local out
out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' )
check "second alert inside cooldown suppressed" 1 "$out"
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
check "recovery notice sent once" 1 "$out"
out=$(clear_alert testkey "recovered" | grep -c '^MAIL\[')
check "recovery not repeated" 0 "$out"
rm -rf "$tmp"
[ $fail -eq 0 ] && echo "selftest OK" || { echo "selftest FAILED"; return 1; }
}
if [ "$1" = "selftest" ]; then selftest; exit $?; fi
#######################################
#######################################
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
if [[ $IDRAC_HOST == "local" ]]; then
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
exit 1
fi
IDRAC_LOGIN_STRING='open'
else
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
fi
#######################################
#######################################
# This script ONLY runs on Dell Servers
get_Dell_server_model
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
echo "/!\ Your server isn't a Dell product. Exiting." >&2
exit 1
fi
#######################################
#######################################
# Prepare, format and define initial variables
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
if [[ $LOW_FAN_SPEED == 0x* ]]; then
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
else
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
fi
#######################################
#######################################
# This returns the lowest temp between CPU and GPU highest thresholds
# Used to calcuate fan speed interpolation
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
else
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
fi
#######################################
#######################################
# Determine where to get CPU temps from iDRAC
# If server model is Gen 14 (*40) or newer
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
DELL_POWEREDGE_GEN_14_OR_NEWER=true
CPU1_TEMPERATURE_INDEX=2
CPU2_TEMPERATURE_INDEX=4
else
DELL_POWEREDGE_GEN_14_OR_NEWER=false
CPU1_TEMPERATURE_INDEX=1
CPU2_TEMPERATURE_INDEX=2
fi
#######################################
#######################################
# Check if sensors are present on the server
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
if [ -z "$EXHAUST_TEMPERATURE" ]; then
echo "No exhaust temperature sensor detected."
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
fi
if [ -z "$CPU2_TEMPERATURE" ]; then
echo "No CPU2 temperature sensor detected."
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
fi
#######################################
#######################################
# Read every drive's own temperature limit once - they do not change, and
# smartctl -x is far heavier than the -A used on the fast loop.
cache_disk_limits
echo "Monitoring $DISK_COUNT_EXPECTED disk(s)."
LAST_SMART_CHECK=0
APPLIED_FAN_SPEED=""
APPLIED_FAN_SPEED_AT=0
#######################################
#######################################
# One measurement pass: read everything, decide a fan speed, raise alarms.
one_pass() {
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
retrieve_gpu_temperature
retrieve_disk_temperatures
# Get highest CPU Temp and check if any of them are over heating
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
if [ "$CPU2_TEMPERATURE" -gt "$CPU1_TEMPERATURE" ]; then
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
fi
fi
if [ "$HIGHEST_CPU_TEMPERATURE" -gt "$CPU_TEMPERATURE_THRESHOLD" ]; then
# CPU is overheating - hand cooling back to Dell and say so out loud.
apply_Dell_fan_control_profile
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
raise_alert cpu "Fan control disengaged on $(hostname)" \
"CPU at ${HIGHEST_CPU_TEMPERATURE}c exceeds the ${CPU_TEMPERATURE_THRESHOLD}c threshold. Dell's default profile has been restored."
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" 0
return
fi
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=false
clear_alert cpu "Fan control restored on $(hostname)"
# CPU and GPU share one curve; disks are interpolated against their own limits
# in retrieve_disk_temperatures. Each source asks for a speed, loudest wins -
# temperatures from different classes of hardware are not comparable directly.
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
if [ "$GPU_TEMPERATURE" -gt "$HIGHEST_CPU_TEMPERATURE" ]; then
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
fi
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_TEMPERATURE" \
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
else
CPU_GPU_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
fi
DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED
FAN_SPEED_DRIVER="cpu/gpu"
if [ "$DISK_FAN_SPEED" -gt "$DECIMAL_CURRENT_FAN_SPEED" ]; then
DECIMAL_CURRENT_FAN_SPEED=$DISK_FAN_SPEED
FAN_SPEED_DRIVER="disk"
fi
# Hold the current speed unless the change is worth making - see the deadband
# note in monitor.sh. Chasing every degree is what made this louder than stock.
local held=""
if should_apply_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED" "$APPLIED_FAN_SPEED_AT"; then
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
APPLIED_FAN_SPEED=$DECIMAL_CURRENT_FAN_SPEED
APPLIED_FAN_SPEED_AT=$(date +%s)
else
held=" held, want $DECIMAL_CURRENT_FAN_SPEED"
fi
COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)"
# Record what the fans are actually doing, not what was merely requested.
record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$APPLIED_FAN_SPEED"
check_alarms
}
check_alarms() {
# A drive within DISK_ALARM_OFFSET of its own limit: fans are already flat out
# for it and it is still climbing.
if [ "$HOTTEST_DISK_TEMPERATURE" -gt 0 ] &&
[ "$HOTTEST_DISK_TEMPERATURE" -ge $((HOTTEST_DISK_LIMIT - DISK_ALARM_OFFSET)) ]; then
raise_alert disk_temp "Disk over temperature on $(hostname)" \
"$HOTTEST_DISK_DEVICE at ${HOTTEST_DISK_TEMPERATURE}c, its own limit is ${HOTTEST_DISK_LIMIT}c. Fans at ${DECIMAL_CURRENT_FAN_SPEED}%."
else
clear_alert disk_temp "Disk temperature normal on $(hostname)"
fi
if [ "$GPU_TEMPERATURE" -gt "$GPU_TEMPERATURE_THRESHOLD" ]; then
raise_alert gpu "GPU over temperature on $(hostname)" \
"GPU at ${GPU_TEMPERATURE}c exceeds the ${GPU_TEMPERATURE_THRESHOLD}c threshold."
else
clear_alert gpu "GPU temperature normal on $(hostname)"
fi
# Climbing steadily while still under every threshold - a dying fan or a
# blocked intake looks exactly like this long before anything crosses a limit.
local cpu_rise disk_rise
cpu_rise=$(rise_over_window 2)
disk_rise=$(rise_over_window 4)
if [ "$cpu_rise" -ge "$TREND_RISE_ALARM" ] || [ "$disk_rise" -ge "$TREND_RISE_ALARM" ]; then
raise_alert trend "Temperature climbing on $(hostname)" \
"Over the last $((TREND_SAMPLES * CHECK_INTERVAL / 60)) minutes: CPU +${cpu_rise}c, hottest disk +${disk_rise}c. Nothing has crossed a threshold yet. Check airflow and fans."
else
clear_alert trend "Temperature stabilised on $(hostname)"
fi
# A drive that stops answering is either asleep or gone. Worth knowing which.
if [ "$DISKS_READ" -lt "$DISK_COUNT_EXPECTED" ]; then
raise_alert disk_count "Disk missing on $(hostname)" \
"Read $DISKS_READ of $DISK_COUNT_EXPECTED disks. A drive is in standby, has dropped off the bus, or has failed."
else
clear_alert disk_count "All disks reporting on $(hostname)"
fi
# SMART is expensive across two dozen drives - hourly, not every loop.
local now; now=$(date +%s)
if [ $((now - LAST_SMART_CHECK)) -ge "$SMART_CHECK_INTERVAL" ]; then
LAST_SMART_CHECK=$now
check_disk_health
fi
}
#######################################
case "${1:-run}" in
once)
one_pass
echo "$COMMENT"
log_line "$COMMENT"
;;
disks)
for dev in $(disk_devices); do
printf '%-12s %-4s limit=%sc\n' "$dev" "$(disk_temperature "$dev")c" "$(disk_limit "$dev")"
done
;;
run)
while true; do
sleep $CHECK_INTERVAL &
SLEEP_PROCESS_PID=$!
one_pass
log_line "$COMMENT"
wait $SLEEP_PROCESS_PID
done
;;
*)
echo "usage: $0 [run|once|disks|selftest]" >&2
exit 1
;;
esac
Reference in New Issue View Git Blame Copy Permalink