Hold fan speed unless the change is worth making

The setpoint was a continuous function of instantaneous temperature and
was pushed to the BMC every pass, so ~1% of fan per degree of CPU noise
meant the fans never settled. Measured on iz-pve0: stock iDRAC held one
speed for 5 minutes through 57-59c jitter while this script made 7
changes in 16 minutes off the same signal.

Adds a 5% deadband around the applied speed, with full-speed requests
never held back and a periodic re-assert in case the BMC forgets.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

README.md

@@ -24,6 +24,19 @@ speed = max( interpolate(hottest of CPU/GPU, 45 -> 75),
 Sources are never compared as raw temperatures — an 85c CPU and a 45c disk are both
 "fine", and a single `max()` over the numbers would be meaningless.
 
+The result is then **held unless it moves by `FAN_SPEED_DEADBAND` (5%)**. This matters
+more than it sounds. The CPU curve is roughly 1% of fan per degree and idle CPU noise is
+±2c, so without a deadband the setpoint changes on almost every pass and the fans never
+settle — measurably worse than stock. Measured on iz-pve0: stock iDRAC held one speed for
+5 minutes straight through 57→59c jitter, while this script made 7 distinct changes in 16
+minutes off the same signal. A request for full speed is never held back, and the held
+value is re-pushed every `FAN_REASSERT_INTERVAL` in case the BMC forgets it.
+
+Note that iDRAC's own profile regulates to a target *RPM* (closed loop, rock steady),
+while manual control sets a *PWM* percentage (open loop, so measured RPM still wanders a
+percent or two at a fixed setpoint). That residual is inherent to manual control, not
+something the deadband can remove.
+
 ### Why disks are measured against their own limits
 
 Every drive reports its own maximum operating temperature (SATA: `Min/Max Temperature
@@ -118,6 +131,8 @@ systemctl enable --now fan_speed.service
 |---|---|---|
 | `CHECK_INTERVAL` | `10` | Seconds between passes |
 | `LOW_FAN_SPEED` / `HIGH_FAN_SPEED` | `18` / `50` | Percent |
+| `FAN_SPEED_DEADBAND` | `5` | Hold the current speed until the request moves this far |
+| `FAN_REASSERT_INTERVAL` | `300` | Re-push the held speed this often regardless |
 | `LOW_TEMPERATURE_THRESHOLD` | `45` | CPU/GPU ramp start |
 | `CPU_TEMPERATURE_THRESHOLD` | `90` | Above this, Dell's profile takes over |
 | `GPU_TEMPERATURE_THRESHOLD` | `75` | |

fan_speed.sh

@@ -82,6 +82,17 @@ selftest() {
   echo "0,50,0,38,20" >> "$TREND_FILE"
   check "partial window stays silent" 0 "$(rise_over_window 4)"
 
+  echo "fan speed deadband (hold unless the change is worth making):"
+  local now; now=$(date +%s)
+  HIGH_FAN_SPEED=50
+  should_apply_fan_speed 21 20 "$now" && check "1% drift holds" hold apply || check "1% drift holds" hold hold
+  should_apply_fan_speed 23 20 "$now" && check "3% drift holds" hold apply || check "3% drift holds" hold hold
+  should_apply_fan_speed 25 20 "$now" && check "5% rise applies" apply apply || check "5% rise applies" apply hold
+  should_apply_fan_speed 15 20 "$now" && check "5% fall applies" apply apply || check "5% fall applies" apply hold
+  should_apply_fan_speed 50 20 "$now" && check "full speed always applies" apply apply || check "full speed always applies" apply hold
+  should_apply_fan_speed 21 "" "$now" && check "first pass always applies" apply apply || check "first pass always applies" apply hold
+  should_apply_fan_speed 21 20 $((now - 400)) && check "stale setpoint is reasserted" apply apply || check "stale setpoint is reasserted" apply hold
+
   echo "alert rate limiting:"
   local out
   out=$( { raise_alert testkey "first" "body"; raise_alert testkey "second" "body"; } | grep -c '^MAIL\[' )
@@ -180,6 +191,8 @@ fi
 cache_disk_limits
 echo "Monitoring $DISK_COUNT_EXPECTED disk(s)."
 LAST_SMART_CHECK=0
+APPLIED_FAN_SPEED=""
+APPLIED_FAN_SPEED_AT=0
 #######################################
 
 #######################################
@@ -232,11 +245,21 @@ one_pass() {
     FAN_SPEED_DRIVER="disk"
   fi
 
+  # Hold the current speed unless the change is worth making - see the deadband
+  # note in monitor.sh. Chasing every degree is what made this louder than stock.
+  local held=""
+  if should_apply_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED" "$APPLIED_FAN_SPEED_AT"; then
     apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
+    APPLIED_FAN_SPEED=$DECIMAL_CURRENT_FAN_SPEED
+    APPLIED_FAN_SPEED_AT=$(date +%s)
+  else
+    held=" held, want $DECIMAL_CURRENT_FAN_SPEED"
+  fi
 
-  COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED($FAN_SPEED_DRIVER)"
+  COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)"
 
-  record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$DECIMAL_CURRENT_FAN_SPEED"
+  # Record what the fans are actually doing, not what was merely requested.
+  record_sample "$HIGHEST_CPU_TEMPERATURE" "$GPU_TEMPERATURE" "$HOTTEST_DISK_TEMPERATURE" "$APPLIED_FAN_SPEED"
   check_alarms
 }
 

monitor.sh

@@ -38,6 +38,34 @@ DISK_ALARM_OFFSET=${DISK_ALARM_OFFSET:-5}
 HDD_LIMIT_CAP=${HDD_LIMIT_CAP:-60}
 SSD_LIMIT_CAP=${SSD_LIMIT_CAP:-70}
 
+# The fan speed is only pushed to the BMC when it moves by at least this much.
+#
+# Without it the setpoint is a continuous function of instantaneous temperature:
+# the CPU curve is ~1% of fan per degree, idle CPU noise is +/-2c, so the fan
+# gets a new speed every single pass and never settles. Measured on iz-pve0,
+# stock iDRAC held one speed for 5 minutes straight through the same jitter
+# while this script made 7 changes in 16 minutes.
+FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-5}
+
+# Re-push the held speed this often anyway, in case the BMC forgets it. Not
+# measured - the MD1200 EMM does forget, iDRAC is believed not to, and one
+# ipmitool call every 5 minutes is cheaper than finding out the hard way.
+FAN_REASSERT_INTERVAL=${FAN_REASSERT_INTERVAL:-300}
+
+# should_apply_fan_speed <wanted> <currently applied> <epoch it was applied>
+# True when the change is worth making. A request for full speed is never held
+# back, and the first pass always applies.
+should_apply_fan_speed() {
+  local wanted=$1 applied=$2 applied_at=$3 delta now
+  [ -n "$applied" ] || return 0
+  [ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0
+  now=$(date +%s)
+  [ $((now - applied_at)) -ge "$FAN_REASSERT_INTERVAL" ] && return 0
+  delta=$((wanted - applied))
+  [ "$delta" -lt 0 ] && delta=$((-delta))
+  [ "$delta" -ge "$FAN_SPEED_DEADBAND" ]
+}
+
 ALERT_EMAIL=${ALERT_EMAIL:-Servers@ntfy1.izebra.xyz}
 ALERT_COOLDOWN=${ALERT_COOLDOWN:-3600}
 STATE_DIR=${STATE_DIR:-/root/fan_speed/state}