From 9961e972df7dc3581ac364fbfa7862651120112b Mon Sep 17 00:00:00 2001 From: Zeb Hering Date: Sun, 30 Aug 2026 16:52:12 -0700 Subject: [PATCH] Move the shelf ramp above the drives' working range, add a deadband Retiring 8x2TB for 4x8TB + a 16TB moved the hottest drive to 40-41c, exactly the old 40c ramp start, and the fan flipped 10<->15% every 30 seconds. The ramp is now 44-56c, which puts the drives back on the floor with 19c of headroom to their 60c limit. Adds the deadband from fan_speed.sh so the shelf does not chase every degree once drives do climb into the ramp, with a downward-only periodic settle so damping cannot leave it permanently loud. selftest and disks no longer take the serial lock - they do not open the port, and refusing to run them while the service is up is unhelpful. The new tests caught a real bug: in `local a=$1 b=$a` bash expands the right-hand sides before the locals exist, so the held value was silently empty. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 22 ++++++++++++--- das_fanctl.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c5d820a..fdb1905 100644 --- a/README.md +++ b/README.md @@ -113,14 +113,28 @@ All knobs are at the top of `das_fanctl.sh`: | `PORT` | `/dev/ttyS1` | Serial device the service cable lands on | | `CHECK_INTERVAL` | `30` | Seconds between temperature readings | | `KEEPALIVE_INTERVAL` | `5` | How often to re-send `_shutup` | -| `DISK_LOW_TEMPERATURE_THRESHOLD` | `40` | At/below this, run at `LOW_FAN_SPEED` | -| `DISK_HIGH_TEMPERATURE_THRESHOLD` | `50` | At/above this, run at `HIGH_FAN_SPEED` | +| `DISK_LOW_TEMPERATURE_THRESHOLD` | `44` | At/below this, run at `LOW_FAN_SPEED` | +| `DISK_HIGH_TEMPERATURE_THRESHOLD` | `56` | At/above this, run at `HIGH_FAN_SPEED` | +| `FAN_SPEED_DEADBAND` | `8` | Hold the current speed until the request moves this far | +| `SETTLE_INTERVAL` | `300` | Allow a downward settle this often | | `LOW_FAN_SPEED` | `10` | Percent | | `HIGH_FAN_SPEED` | `60` | Percent | | `BP_*_THRESHOLD` | `30` / `45` | Fallback curve, backplane air sensors | -The defaults were sized against an idle spread of 32–39c across nine mixed SATA/SAS -drives. Your airflow is not that airflow — watch a day of log before trusting them. +**Start the ramp above the drives' normal working range.** This is the single most +common way to make a shelf cycle audibly: if the ramp starts inside the range the drives +already occupy, ordinary drift crosses it and the fans flip every cycle. The curve is +~4%/degC, so one degree moves the fan noticeably. + +Worked example: the original 40–50c was sized for 8×2TB drives idling at 32–39c. Retiring +those for 4×8TB + a 16TB moved the hottest drive to 40–41c — exactly the old ramp start — +and the fan flipped 10↔15% every 30 seconds (18 samples at 10, 22 at 15). Moving the ramp +to 44–56c put the drives back on the floor: 11 consecutive samples at 10%, and they still +have 19c of headroom to their 60c limit. + +`FAN_SPEED_DEADBAND` damps what remains, so once drives do climb into the ramp it does not +chase every degree. A settle is permitted every `SETTLE_INTERVAL`, downward only — +a symmetric deadband would catch the fan on the way up and never let it back down. `KEEPALIVE_INTERVAL` is inherited from the 5s cadence the original setup used, not measured. How fast the EMM actually reverts is unknown; if the fans audibly surge diff --git a/das_fanctl.sh b/das_fanctl.sh index aa28e75..aa3542c 100755 --- a/das_fanctl.sh +++ b/das_fanctl.sh @@ -21,20 +21,59 @@ HIGH_FAN_SPEED=60 # internal drives sit on pci-0000:02:00.0 while the shelf is on 04:00.0. Name the # shelf's HBA explicitly on any host with more than one. DISK_GLOB=${DISK_GLOB:-/dev/disk/by-path/*-sas-exp*-lun-0} -DISK_LOW_TEMPERATURE_THRESHOLD=40 # at/below this -> LOW_FAN_SPEED (idle spread is 32-39c) -DISK_HIGH_TEMPERATURE_THRESHOLD=50 # at/above this -> HIGH_FAN_SPEED +# The ramp must START ABOVE the drives' normal working range, or every degree of +# ordinary drift crosses it and the fans cycle audibly. The 4x8TB + 1x16TB set in +# this shelf works at 30-41c against a 60c limit, so 44 leaves 3c of clearance. +# The previous 40-50c was sized for an 8x2TB set that idled at 32-39c; the newer +# drives run warmer and sat exactly on the old ramp start, flipping 10<->15% +# every 30 seconds. +DISK_LOW_TEMPERATURE_THRESHOLD=${DISK_LOW_TEMPERATURE_THRESHOLD:-44} +DISK_HIGH_TEMPERATURE_THRESHOLD=${DISK_HIGH_TEMPERATURE_THRESHOLD:-56} # Backplane air sensors, only used when no disk will answer (all spun down, smartctl gone) BP_LOW_TEMPERATURE_THRESHOLD=30 BP_HIGH_TEMPERATURE_THRESHOLD=45 +# Hold the current speed unless the request moves this far. The curve is ~4%/degC +# even after widening, so without damping a single degree of drive drift moves the +# fan. A re-settle is allowed periodically, downward only - see settle_fan_speed. +FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-8} +SETTLE_INTERVAL=${SETTLE_INTERVAL:-300} + SPEED=$LOW_FAN_SPEED +SPEED_SET_AT=0 -# Only one talker on the serial line (detach screen/attachDAS.sh first) -exec 9>/run/das_fanctl.lock -flock -n 9 || { echo "another das_fanctl is running (or /dev/ttyS1 is busy)" >&2; exit 1; } +# Only one talker on the serial line (detach screen/attachDAS.sh first). Taken +# only by the subcommands that actually open the port - selftest and disks do +# not, and refusing to run them while the service is up is just unhelpful. +claim_serial_port() { + exec 9>/run/das_fanctl.lock + flock -n 9 || { echo "another das_fanctl is running (or $PORT is busy)" >&2; exit 1; } + stty -F $PORT 38400 raw -echo -echoe -echok -echoctl -echoke +} -stty -F $PORT 38400 raw -echo -echoe -echok -echoctl -echoke +# True when the request has moved far enough to act on. Full speed is never held back. +fan_speed_changed_enough() { + local wanted=$1 applied=$2 delta + [ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0 + delta=$((wanted - applied)); [ "$delta" -lt 0 ] && delta=$((-delta)) + [ "$delta" -ge "$FAN_SPEED_DEADBAND" ] +} + +# settle_fan_speed +# A symmetric deadband catches the fan on the way up and never lets it back down, +# which leaves the shelf permanently louder than it needs to be. So periodically - +# and only downward - let it settle to what is actually wanted. Upward moves always +# require a full deadband crossing. +settle_fan_speed() { + local wanted=$1 applied=$2 due=$3 + # Declared separately on purpose: in `local a=$1 b=$a` the right-hand sides are + # expanded before the locals exist, so b would get the OUTER scope's a (empty). + local target=$applied + fan_speed_changed_enough "$wanted" "$applied" && target=$wanted + if [ "$due" = 1 ] && [ "$wanted" -lt "$target" ]; then target=$wanted; fi + echo "$target" +} # Fire and forget - the DAS answers with a bare prompt, nothing worth reading set_fan() { printf '_shutup %s\r' "$1" > $PORT; } @@ -89,10 +128,17 @@ one_pass() { low=$BP_LOW_TEMPERATURE_THRESHOLD; high=$BP_HIGH_TEMPERATURE_THRESHOLD fi - SPEED=$(calculate_interpolated_fan_speed "$temp" $low $high $LOW_FAN_SPEED $HIGH_FAN_SPEED) + local want now due=0 prev=$SPEED held="" + want=$(calculate_interpolated_fan_speed "$temp" $low $high $LOW_FAN_SPEED $HIGH_FAN_SPEED) + now=$(date +%s) + [ $((now - SPEED_SET_AT)) -ge "$SETTLE_INTERVAL" ] && due=1 + SPEED=$(settle_fan_speed "$want" "$SPEED" "$due") + { [ "$SPEED" != "$prev" ] || [ "$due" = 1 ]; } && SPEED_SET_AT=$now + [ "$SPEED" = "$want" ] || held=" held, want $want" + set_fan "$SPEED" - echo "$(date +'%Y-%m-%d %H:%M:%S') || DAS ${source}:${temp}c | Fan Speed:$SPEED" >> $LOG - echo "DAS ${source}:${temp}c -> fan ${SPEED}%" + echo "$(date +'%Y-%m-%d %H:%M:%S') || DAS ${source}:${temp}c | Fan Speed:${SPEED}${held}" >> $LOG + echo "DAS ${source}:${temp}c -> fan ${SPEED}%${held}" } selftest() { @@ -107,9 +153,21 @@ selftest() { [ "$(calculate_interpolated_fan_speed 55 40 50 10 60)" = 60 ] || { echo "FAIL: high clamp"; exit 1; } [ "$(calculate_interpolated_fan_speed 45 40 50 10 60)" = 35 ] || { echo "FAIL: midpoint"; exit 1; } [ "$(hottest_disk)" -gt 0 ] || { echo "FAIL: no disk temperatures readable"; exit 1; } + HIGH_FAN_SPEED=60 + [ "$(settle_fan_speed 15 10 0)" = 10 ] || { echo "FAIL: 5% drift should hold"; exit 1; } + [ "$(settle_fan_speed 18 10 0)" = 18 ] || { echo "FAIL: 8% rise should be adopted"; exit 1; } + [ "$(settle_fan_speed 60 10 0)" = 60 ] || { echo "FAIL: full speed always adopted"; exit 1; } + [ "$(settle_fan_speed 26 30 0)" = 30 ] || { echo "FAIL: small fall should hold"; exit 1; } + [ "$(settle_fan_speed 26 30 1)" = 26 ] || { echo "FAIL: should settle down when due"; exit 1; } + [ "$(settle_fan_speed 34 30 1)" = 30 ] || { echo "FAIL: must not drift up on a settle"; exit 1; } echo "selftest OK" } +case "${1:-loop}" in + selftest|disks) ;; # no serial access + *) claim_serial_port ;; +esac + case "${1:-loop}" in temps) das_cmd _temp_rd ;; disks) for l in $DISK_GLOB; do