From 56ae7542a2c142850e8a60f46feee97c809e1efe Mon Sep 17 00:00:00 2001 From: Zeb Hering Date: Fri, 28 Aug 2026 20:07:33 -0700 Subject: [PATCH] Hold fan speed unless the change is worth making The setpoint was a continuous function of instantaneous temperature and was pushed to the BMC every pass, so ~1% of fan per degree of CPU noise meant the fans never settled. Measured on iz-pve0: stock iDRAC held one speed for 5 minutes through 57-59c jitter while this script made 7 changes in 16 minutes off the same signal. Adds a 5% deadband around the applied speed, with full-speed requests never held back and a periodic re-assert in case the BMC forgets. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 15 +++++++++++++++ fan_speed.sh | 29 ++++++++++++++++++++++++++--- monitor.sh | 28 ++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 618a1b1..232f572 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,19 @@ speed = max( interpolate(hottest of CPU/GPU, 45 -> 75), Sources are never compared as raw temperatures — an 85c CPU and a 45c disk are both "fine", and a single `max()` over the numbers would be meaningless. +The result is then **held unless it moves by `FAN_SPEED_DEADBAND` (5%)**. This matters +more than it sounds. The CPU curve is roughly 1% of fan per degree and idle CPU noise is +±2c, so without a deadband the setpoint changes on almost every pass and the fans never +settle — measurably worse than stock. Measured on iz-pve0: stock iDRAC held one speed for +5 minutes straight through 57→59c jitter, while this script made 7 distinct changes in 16 +minutes off the same signal. A request for full speed is never held back, and the held +value is re-pushed every `FAN_REASSERT_INTERVAL` in case the BMC forgets it. + +Note that iDRAC's own profile regulates to a target *RPM* (closed loop, rock steady), +while manual control sets a *PWM* percentage (open loop, so measured RPM still wanders a +percent or two at a fixed setpoint). That residual is inherent to manual control, not +something the deadband can remove. + ### Why disks are measured against their own limits Every drive reports its own maximum operating temperature (SATA: `Min/Max Temperature @@ -118,6 +131,8 @@ systemctl enable --now fan_speed.service |---|---|---| | `CHECK_INTERVAL` | `10` | Seconds between passes | | `LOW_FAN_SPEED` / `HIGH_FAN_SPEED` | `18` / `50` | Percent | +| `FAN_SPEED_DEADBAND` | `5` | Hold the current speed until the request moves this far | +| `FAN_REASSERT_INTERVAL` | `300` | Re-push the held speed this often regardless | | `LOW_TEMPERATURE_THRESHOLD` | `45` | CPU/GPU ramp start | | `CPU_TEMPERATURE_THRESHOLD` | `90` | Above this, Dell's profile takes over | | `GPU_TEMPERATURE_THRESHOLD` | `75` | | diff --git a/fan_speed.sh b/fan_speed.sh index efdc6fc..3793aab 100755 --- a/fan_speed.sh +++ b/fan_speed.sh @@ -82,6 +82,17 @@ selftest() { echo "0,50,0,38,20" >> "$TREND_FILE" check "partial window stays silent" 0 "$(rise_over_window 4)" + echo "fan speed deadband (hold unless the change is worth making):" + local now; now=$(date +%s) + HIGH_FAN_SPEED=50 + should_apply_fan_speed 21 20 "$now" && check "1% drift holds" hold apply || check "1% drift holds" hold hold + should_apply_fan_speed 23 20 "$now" && check "3% drift holds" hold apply || check "3% drift holds" hold hold + should_apply_fan_speed 25 20 "$now" && check "5% rise applies" apply apply || check "5% rise applies" apply hold + should_apply_fan_speed 15 20 "$now" && check "5% fall applies" apply apply || check "5% fall applies" apply hold + should_apply_fan_speed 50 20 "$now" && check "full speed always applies" apply apply || check "full speed always applies" apply hold + should_apply_fan_speed 21 "" "$now" && check "first pass always applies" apply apply || check "first pass always applies" apply hold + should_apply_fan_speed 21 20 $((now - 400)) && check "stale setpoint is reasserted" apply apply || check "stale setpoint is reasserted" apply hold + echo "alert rate limiting:" local out out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' ) @@ -180,6 +191,8 @@ fi cache_disk_limits echo "Monitoring $DISK_COUNT_EXPECTED disk(s)." LAST_SMART_CHECK=0 +APPLIED_FAN_SPEED="" +APPLIED_FAN_SPEED_AT=0 ####################################### ####################################### @@ -232,11 +245,21 @@ one_pass() { FAN_SPEED_DRIVER="disk" fi - apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED" + # Hold the current speed unless the change is worth making - see the deadband + # note in monitor.sh. Chasing every degree is what made this louder than stock. + local held="" + if should_apply_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED" "$APPLIED_FAN_SPEED_AT"; then + apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED" + APPLIED_FAN_SPEED=$DECIMAL_CURRENT_FAN_SPEED + APPLIED_FAN_SPEED_AT=$(date +%s) + else + held=" held, want $DECIMAL_CURRENT_FAN_SPEED" + fi - COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED($FAN_SPEED_DRIVER)" + COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)" - record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$DECIMAL_CURRENT_FAN_SPEED" + # Record what the fans are actually doing, not what was merely requested. + record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$APPLIED_FAN_SPEED" check_alarms } diff --git a/monitor.sh b/monitor.sh index e0bda3d..e7e584c 100644 --- a/monitor.sh +++ b/monitor.sh @@ -38,6 +38,34 @@ DISK_ALARM_OFFSET=${DISK_ALARM_OFFSET:-5} HDD_LIMIT_CAP=${HDD_LIMIT_CAP:-60} SSD_LIMIT_CAP=${SSD_LIMIT_CAP:-70} +# The fan speed is only pushed to the BMC when it moves by at least this much. +# +# Without it the setpoint is a continuous function of instantaneous temperature: +# the CPU curve is ~1% of fan per degree, idle CPU noise is +/-2c, so the fan +# gets a new speed every single pass and never settles. Measured on iz-pve0, +# stock iDRAC held one speed for 5 minutes straight through the same jitter +# while this script made 7 changes in 16 minutes. +FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-5} + +# Re-push the held speed this often anyway, in case the BMC forgets it. Not +# measured - the MD1200 EMM does forget, iDRAC is believed not to, and one +# ipmitool call every 5 minutes is cheaper than finding out the hard way. +FAN_REASSERT_INTERVAL=${FAN_REASSERT_INTERVAL:-300} + +# should_apply_fan_speed +# True when the change is worth making. A request for full speed is never held +# back, and the first pass always applies. +should_apply_fan_speed() { + local wanted=$1 applied=$2 applied_at=$3 delta now + [ -n "$applied" ] || return 0 + [ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0 + now=$(date +%s) + [ $((now - applied_at)) -ge "$FAN_REASSERT_INTERVAL" ] && return 0 + delta=$((wanted - applied)) + [ "$delta" -lt 0 ] && delta=$((-delta)) + [ "$delta" -ge "$FAN_SPEED_DEADBAND" ] +} + ALERT_EMAIL=${ALERT_EMAIL:-Servers@ntfy1.izebra.xyz} ALERT_COOLDOWN=${ALERT_COOLDOWN:-3600} STATE_DIR=${STATE_DIR:-/root/fan_speed/state}