Calibrate the CPU ramp per host and fix the deadband leak
Two problems found by A/B against Dell's own profile on an R720xd, 8 minute windows at matched CPU temperature: The CPU ramp started at a fixed 45c, calibrated on a chassis idling at 48c. On one idling at 58c that sat 40% up the ramp doing nothing useful and ran 6878 RPM against stock's 6240. The ramp is now derived from the CPU sensor's own upper-non-critical threshold, and the fan floor is per-host since PWM->RPM is chassis specific. The 5 minute re-assert re-pushed the current request rather than the held value, so the deadband leaked a few percent every interval and the setpoint drifted 32 -> 36 -> 31. next_fan_speed now separates "has it moved enough to adopt" from "is it time to re-push". Also documents that running above stock is correct on hosts whose drives need it: removing the drives' vote on iz-pve0 hit stock RPM exactly and took two rear-bay SSDs from 54c to 61c in minutes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
95
fan_speed.sh
95
fan_speed.sh
@@ -27,15 +27,37 @@ IDRAC_HOST=local
|
||||
IDRAC_USERNAME=root
|
||||
IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin}
|
||||
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
|
||||
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
|
||||
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
|
||||
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
|
||||
LOW_FAN_SPEED=18 #only decimal numbers
|
||||
HIGH_FAN_SPEED=50 #only decimal numbers
|
||||
# Per-chassis: the PWM that matches what Dell's own profile runs at idle. The
|
||||
# PWM->RPM relationship is chassis specific, so this is measured, not guessed -
|
||||
# see "Calibrating a host" in the README. 18/50 suits an R730xd that idles at
|
||||
# 48c; an R720xd idling at 60c needs a floor of 28 to match stock.
|
||||
LOW_FAN_SPEED=${LOW_FAN_SPEED:-18}
|
||||
HIGH_FAN_SPEED=${HIGH_FAN_SPEED:-50}
|
||||
TABLE_HEADER_PRINT_INTERVAL=2
|
||||
|
||||
# The CPU ramp is derived from the CPU sensor's own upper-non-critical
|
||||
# threshold as reported by iDRAC, exactly like the disks are derived from
|
||||
# theirs. A fixed 45c start was calibrated on a chassis that idles at 48c; on
|
||||
# one that idles at 58c it sat 40% up the ramp doing nothing useful, and ran
|
||||
# ~10% more airflow than Dell's own profile chose at the same temperature
|
||||
# (6878 vs 6240 RPM measured over 8 minutes on an R720xd).
|
||||
#
|
||||
# Offsets chosen so a 58c idle lands near what Dell picks. Both an R720xd and
|
||||
# an R730xd report upper-non-critical 77c, so this resolves to a 53->69c ramp.
|
||||
CPU_RAMP_LOW_OFFSET=${CPU_RAMP_LOW_OFFSET:-24}
|
||||
CPU_RAMP_HIGH_OFFSET=${CPU_RAMP_HIGH_OFFSET:-8}
|
||||
CPU_LIMIT_FALLBACK=${CPU_LIMIT_FALLBACK:-77}
|
||||
|
||||
# Disk thresholds, alarm routing and trend window all live in monitor.sh.
|
||||
|
||||
# iDRAC's declared upper-non-critical temperature for the CPU sensor.
|
||||
cpu_upper_non_critical() {
|
||||
ipmitool -I $IDRAC_LOGIN_STRING sdr get "Temp" 2>/dev/null |
|
||||
awk -F: '/Upper non-critical/ {gsub(/[^0-9.]/,"",$2); print int($2); exit}'
|
||||
}
|
||||
|
||||
#######################################
|
||||
# Self test - pure logic only, no IPMI and no drives touched.
|
||||
selftest() {
|
||||
@@ -83,15 +105,17 @@ selftest() {
|
||||
check "partial window stays silent" 0 "$(rise_over_window 4)"
|
||||
|
||||
echo "fan speed deadband (hold unless the change is worth making):"
|
||||
local now; now=$(date +%s)
|
||||
HIGH_FAN_SPEED=50
|
||||
should_apply_fan_speed 21 20 "$now" && check "1% drift holds" hold apply || check "1% drift holds" hold hold
|
||||
should_apply_fan_speed 23 20 "$now" && check "3% drift holds" hold apply || check "3% drift holds" hold hold
|
||||
should_apply_fan_speed 25 20 "$now" && check "5% rise applies" apply apply || check "5% rise applies" apply hold
|
||||
should_apply_fan_speed 15 20 "$now" && check "5% fall applies" apply apply || check "5% fall applies" apply hold
|
||||
should_apply_fan_speed 50 20 "$now" && check "full speed always applies" apply apply || check "full speed always applies" apply hold
|
||||
should_apply_fan_speed 21 "" "$now" && check "first pass always applies" apply apply || check "first pass always applies" apply hold
|
||||
should_apply_fan_speed 21 20 $((now - 400)) && check "stale setpoint is reasserted" apply apply || check "stale setpoint is reasserted" apply hold
|
||||
check "1% drift holds" 20 "$(next_fan_speed 21 20)"
|
||||
check "3% drift holds" 20 "$(next_fan_speed 23 20)"
|
||||
check "4% drift holds" 32 "$(next_fan_speed 36 32)"
|
||||
check "5% rise is adopted" 25 "$(next_fan_speed 25 20)"
|
||||
check "5% fall is adopted" 15 "$(next_fan_speed 15 20)"
|
||||
check "full speed always adopted" 50 "$(next_fan_speed 50 20)"
|
||||
check "first pass adopts the request" 21 "$(next_fan_speed 21 '')"
|
||||
# A re-assert must re-push what is held, not adopt the current want - that
|
||||
# leak is what let the setpoint drift 32 -> 36 -> 31 on iz-pve0.
|
||||
check "re-assert re-pushes held value" 32 "$(next_fan_speed 34 32)"
|
||||
|
||||
echo "alert rate limiting:"
|
||||
local out
|
||||
@@ -146,14 +170,12 @@ fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# This returns the lowest temp between CPU and GPU highest thresholds
|
||||
# Used to calcuate fan speed interpolation
|
||||
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
|
||||
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
|
||||
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
|
||||
else
|
||||
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
|
||||
fi
|
||||
# Derive the CPU ramp from what iDRAC says this CPU can actually take.
|
||||
CPU_LIMIT=$(cpu_upper_non_critical)
|
||||
[ -n "$CPU_LIMIT" ] || CPU_LIMIT=$CPU_LIMIT_FALLBACK
|
||||
LOW_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_LOW_OFFSET))
|
||||
HIGH_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_HIGH_OFFSET))
|
||||
echo "CPU limit ${CPU_LIMIT}c -> fan ramp ${LOW_TEMPERATURE_THRESHOLD}c..${HIGH_TEMPERATURE_THRESHOLD}c"
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
@@ -227,15 +249,16 @@ one_pass() {
|
||||
# CPU and GPU share one curve; disks are interpolated against their own limits
|
||||
# in retrieve_disk_temperatures. Each source asks for a speed, loudest wins -
|
||||
# temperatures from different classes of hardware are not comparable directly.
|
||||
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
|
||||
if [ "$GPU_TEMPERATURE" -gt "$HIGHEST_CPU_TEMPERATURE" ]; then
|
||||
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
|
||||
fi
|
||||
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
|
||||
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_TEMPERATURE" \
|
||||
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
||||
else
|
||||
CPU_GPU_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
|
||||
CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_CPU_TEMPERATURE" \
|
||||
$LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
||||
# The GPU has its own limit and so gets its own ramp, rather than being
|
||||
# compared against the CPU's on a shared scale.
|
||||
if [ "$GPU_TEMPERATURE" -gt 0 ]; then
|
||||
local gpu_speed
|
||||
gpu_speed=$(calculate_interpolated_fan_speed "$GPU_TEMPERATURE" \
|
||||
$((GPU_TEMPERATURE_THRESHOLD - CPU_RAMP_LOW_OFFSET)) $GPU_TEMPERATURE_THRESHOLD \
|
||||
$LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
||||
[ "$gpu_speed" -gt "$CPU_GPU_FAN_SPEED" ] && CPU_GPU_FAN_SPEED=$gpu_speed
|
||||
fi
|
||||
|
||||
DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED
|
||||
@@ -247,13 +270,15 @@ one_pass() {
|
||||
|
||||
# Hold the current speed unless the change is worth making - see the deadband
|
||||
# note in monitor.sh. Chasing every degree is what made this louder than stock.
|
||||
local held=""
|
||||
if should_apply_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED" "$APPLIED_FAN_SPEED_AT"; then
|
||||
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
|
||||
APPLIED_FAN_SPEED=$DECIMAL_CURRENT_FAN_SPEED
|
||||
APPLIED_FAN_SPEED_AT=$(date +%s)
|
||||
else
|
||||
held=" held, want $DECIMAL_CURRENT_FAN_SPEED"
|
||||
local target now held=""
|
||||
target=$(next_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED")
|
||||
[ "$target" = "$DECIMAL_CURRENT_FAN_SPEED" ] || held=" held, want $DECIMAL_CURRENT_FAN_SPEED"
|
||||
now=$(date +%s)
|
||||
if [ "$target" != "$APPLIED_FAN_SPEED" ] ||
|
||||
[ $((now - APPLIED_FAN_SPEED_AT)) -ge "$FAN_REASSERT_INTERVAL" ]; then
|
||||
apply_user_fan_control "$target"
|
||||
APPLIED_FAN_SPEED=$target
|
||||
APPLIED_FAN_SPEED_AT=$now
|
||||
fi
|
||||
|
||||
COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)"
|
||||
|
||||
Reference in New Issue
Block a user