9961e972df
Retiring 8x2TB for 4x8TB + a 16TB moved the hottest drive to 40-41c, exactly the old 40c ramp start, and the fan flipped 10<->15% every 30 seconds. The ramp is now 44-56c, which puts the drives back on the floor with 19c of headroom to their 60c limit. Adds the deadband from fan_speed.sh so the shelf does not chase every degree once drives do climb into the ramp, with a downward-only periodic settle so damping cannot leave it permanently loud. selftest and disks no longer take the serial lock - they do not open the port, and refusing to run them while the service is up is unhelpful. The new tests caught a real bug: in `local a=$1 b=$a` bash expands the right-hand sides before the locals exist, so the held value was silently empty. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
190 lines
8.1 KiB
Bash
Executable File
190 lines
8.1 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
#Zeb H.
|
|
#Dell MD1200 DAS fan control - read temps, set fan speed to match.
|
|
#Drives the curve off drive temperature, falls back to the enclosure backplane sensors.
|
|
#Usage: das_fanctl.sh [loop|once|temps|disks|selftest] (default: loop)
|
|
|
|
source /root/fan_speed/functions.sh
|
|
|
|
PORT=${PORT:-/dev/ttyS1}
|
|
LOG=/root/fan_speed/log/das_fan.log
|
|
CHECK_INTERVAL=30 # seconds between temperature readings
|
|
KEEPALIVE_INTERVAL=5 # re-send _shutup this often, the DAS forgets it otherwise
|
|
LOW_FAN_SPEED=10
|
|
HIGH_FAN_SPEED=60
|
|
|
|
# Disks in the enclosure - anything behind the DAS SAS expander. The server's own
|
|
# drives hang off pci-...-scsi-* instead, so this glob does not pick them up.
|
|
# Drives in THIS shelf. "-sas-exp" alone is not enough to identify an enclosure:
|
|
# a server's own backplane is often behind an expander too, and on iz-pve0 the
|
|
# internal drives sit on pci-0000:02:00.0 while the shelf is on 04:00.0. Name the
|
|
# shelf's HBA explicitly on any host with more than one.
|
|
DISK_GLOB=${DISK_GLOB:-/dev/disk/by-path/*-sas-exp*-lun-0}
|
|
# The ramp must START ABOVE the drives' normal working range, or every degree of
|
|
# ordinary drift crosses it and the fans cycle audibly. The 4x8TB + 1x16TB set in
|
|
# this shelf works at 30-41c against a 60c limit, so 44 leaves 3c of clearance.
|
|
# The previous 40-50c was sized for an 8x2TB set that idled at 32-39c; the newer
|
|
# drives run warmer and sat exactly on the old ramp start, flipping 10<->15%
|
|
# every 30 seconds.
|
|
DISK_LOW_TEMPERATURE_THRESHOLD=${DISK_LOW_TEMPERATURE_THRESHOLD:-44}
|
|
DISK_HIGH_TEMPERATURE_THRESHOLD=${DISK_HIGH_TEMPERATURE_THRESHOLD:-56}
|
|
|
|
# Backplane air sensors, only used when no disk will answer (all spun down, smartctl gone)
|
|
BP_LOW_TEMPERATURE_THRESHOLD=30
|
|
BP_HIGH_TEMPERATURE_THRESHOLD=45
|
|
|
|
# Hold the current speed unless the request moves this far. The curve is ~4%/degC
|
|
# even after widening, so without damping a single degree of drive drift moves the
|
|
# fan. A re-settle is allowed periodically, downward only - see settle_fan_speed.
|
|
FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-8}
|
|
SETTLE_INTERVAL=${SETTLE_INTERVAL:-300}
|
|
|
|
SPEED=$LOW_FAN_SPEED
|
|
SPEED_SET_AT=0
|
|
|
|
# Only one talker on the serial line (detach screen/attachDAS.sh first). Taken
|
|
# only by the subcommands that actually open the port - selftest and disks do
|
|
# not, and refusing to run them while the service is up is just unhelpful.
|
|
claim_serial_port() {
|
|
exec 9>/run/das_fanctl.lock
|
|
flock -n 9 || { echo "another das_fanctl is running (or $PORT is busy)" >&2; exit 1; }
|
|
stty -F $PORT 38400 raw -echo -echoe -echok -echoctl -echoke
|
|
}
|
|
|
|
# True when the request has moved far enough to act on. Full speed is never held back.
|
|
fan_speed_changed_enough() {
|
|
local wanted=$1 applied=$2 delta
|
|
[ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0
|
|
delta=$((wanted - applied)); [ "$delta" -lt 0 ] && delta=$((-delta))
|
|
[ "$delta" -ge "$FAN_SPEED_DEADBAND" ]
|
|
}
|
|
|
|
# settle_fan_speed <wanted> <applied> <settle due, 0|1>
|
|
# A symmetric deadband catches the fan on the way up and never lets it back down,
|
|
# which leaves the shelf permanently louder than it needs to be. So periodically -
|
|
# and only downward - let it settle to what is actually wanted. Upward moves always
|
|
# require a full deadband crossing.
|
|
settle_fan_speed() {
|
|
local wanted=$1 applied=$2 due=$3
|
|
# Declared separately on purpose: in `local a=$1 b=$a` the right-hand sides are
|
|
# expanded before the locals exist, so b would get the OUTER scope's a (empty).
|
|
local target=$applied
|
|
fan_speed_changed_enough "$wanted" "$applied" && target=$wanted
|
|
if [ "$due" = 1 ] && [ "$wanted" -lt "$target" ]; then target=$wanted; fi
|
|
echo "$target"
|
|
}
|
|
|
|
# Fire and forget - the DAS answers with a bare prompt, nothing worth reading
|
|
set_fan() { printf '_shutup %s\r' "$1" > $PORT; }
|
|
|
|
# Send a command, return whatever the DAS says back
|
|
das_cmd() {
|
|
local out; out=$(mktemp)
|
|
timeout 3 cat $PORT > "$out" &
|
|
local pid=$!
|
|
sleep 0.2
|
|
printf '%s\r' "$1" > $PORT
|
|
wait $pid
|
|
cat "$out"; rm -f "$out"
|
|
}
|
|
|
|
# Hottest drive-backplane sensor from a _temp_rd readout on stdin.
|
|
# ponytail: BP_* only - EXP0 (SAS expander) idles ~59c and would peg the fans,
|
|
# SIM0 is the enclosure module. Add them here if drives ever run cool but the DAS cooks.
|
|
hottest_bp() {
|
|
awk -F'[=c]' '/BP_[0-9]+\[/ {gsub(/ /,"",$2); if ($2+0 > m) m = $2+0} END {print m+0}'
|
|
}
|
|
|
|
# Hottest drive in the enclosure, 0 if nothing answered.
|
|
# -n standby so a sleeping drive is skipped rather than spun up just to be measured.
|
|
# SATA reports attribute 194, SAS reports "Current Drive Temperature".
|
|
hottest_disk() {
|
|
local link dev temp max=0
|
|
for link in $DISK_GLOB; do
|
|
[ -e "$link" ] || continue
|
|
dev=$(readlink -f "$link")
|
|
temp=$(smartctl -n standby -A "$dev" 2>/dev/null |
|
|
awk '/Temperature_Celsius/ {print $10; exit} /Current Drive Temperature/ {print $4; exit}')
|
|
[ -n "$temp" ] && [ "$temp" -gt "$max" ] 2>/dev/null && max=$temp
|
|
done
|
|
echo $max
|
|
}
|
|
|
|
# Read temps, recalculate SPEED, apply it
|
|
one_pass() {
|
|
local disk bp source temp low high
|
|
disk=$(hottest_disk)
|
|
if [ "$disk" -gt 0 ]; then
|
|
source=disk; temp=$disk
|
|
low=$DISK_LOW_TEMPERATURE_THRESHOLD; high=$DISK_HIGH_TEMPERATURE_THRESHOLD
|
|
else
|
|
bp=$(das_cmd _temp_rd | hottest_bp)
|
|
if [ "$bp" -le 0 ] 2>/dev/null; then
|
|
echo "$(date +'%Y-%m-%d %H:%M:%S') || no disk or backplane reading, holding fan at ${SPEED}%" >> $LOG
|
|
return 1
|
|
fi
|
|
source=backplane; temp=$bp
|
|
low=$BP_LOW_TEMPERATURE_THRESHOLD; high=$BP_HIGH_TEMPERATURE_THRESHOLD
|
|
fi
|
|
|
|
local want now due=0 prev=$SPEED held=""
|
|
want=$(calculate_interpolated_fan_speed "$temp" $low $high $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
|
now=$(date +%s)
|
|
[ $((now - SPEED_SET_AT)) -ge "$SETTLE_INTERVAL" ] && due=1
|
|
SPEED=$(settle_fan_speed "$want" "$SPEED" "$due")
|
|
{ [ "$SPEED" != "$prev" ] || [ "$due" = 1 ]; } && SPEED_SET_AT=$now
|
|
[ "$SPEED" = "$want" ] || held=" held, want $want"
|
|
|
|
set_fan "$SPEED"
|
|
echo "$(date +'%Y-%m-%d %H:%M:%S') || DAS ${source}:${temp}c | Fan Speed:${SPEED}${held}" >> $LOG
|
|
echo "DAS ${source}:${temp}c -> fan ${SPEED}%${held}"
|
|
}
|
|
|
|
selftest() {
|
|
local canned=' BP_1[2] = 30c
|
|
BP_2[3] = 44c
|
|
SIM0[0] = 36c
|
|
EXP0[4] = 59c
|
|
AVG = 39c'
|
|
[ "$(printf '%s' "$canned" | hottest_bp)" = 44 ] || { echo "FAIL: bp parser"; exit 1; }
|
|
[ "$(printf '%s' 'garbage' | hottest_bp)" = 0 ] || { echo "FAIL: bp parser on junk"; exit 1; }
|
|
[ "$(calculate_interpolated_fan_speed 40 40 50 10 60)" = 10 ] || { echo "FAIL: low clamp"; exit 1; }
|
|
[ "$(calculate_interpolated_fan_speed 55 40 50 10 60)" = 60 ] || { echo "FAIL: high clamp"; exit 1; }
|
|
[ "$(calculate_interpolated_fan_speed 45 40 50 10 60)" = 35 ] || { echo "FAIL: midpoint"; exit 1; }
|
|
[ "$(hottest_disk)" -gt 0 ] || { echo "FAIL: no disk temperatures readable"; exit 1; }
|
|
HIGH_FAN_SPEED=60
|
|
[ "$(settle_fan_speed 15 10 0)" = 10 ] || { echo "FAIL: 5% drift should hold"; exit 1; }
|
|
[ "$(settle_fan_speed 18 10 0)" = 18 ] || { echo "FAIL: 8% rise should be adopted"; exit 1; }
|
|
[ "$(settle_fan_speed 60 10 0)" = 60 ] || { echo "FAIL: full speed always adopted"; exit 1; }
|
|
[ "$(settle_fan_speed 26 30 0)" = 30 ] || { echo "FAIL: small fall should hold"; exit 1; }
|
|
[ "$(settle_fan_speed 26 30 1)" = 26 ] || { echo "FAIL: should settle down when due"; exit 1; }
|
|
[ "$(settle_fan_speed 34 30 1)" = 30 ] || { echo "FAIL: must not drift up on a settle"; exit 1; }
|
|
echo "selftest OK"
|
|
}
|
|
|
|
case "${1:-loop}" in
|
|
selftest|disks) ;; # no serial access
|
|
*) claim_serial_port ;;
|
|
esac
|
|
|
|
case "${1:-loop}" in
|
|
temps) das_cmd _temp_rd ;;
|
|
disks) for l in $DISK_GLOB; do
|
|
d=$(readlink -f "$l")
|
|
printf '%-12s %s\n' "$d" "$(smartctl -n standby -A "$d" 2>/dev/null |
|
|
awk '/Temperature_Celsius/ {print $10"c"; exit} /Current Drive Temperature/ {print $4"c"; exit}')"
|
|
done; echo "hottest: $(hottest_disk)c" ;;
|
|
once) one_pass ;;
|
|
selftest) selftest ;;
|
|
loop) while true; do
|
|
one_pass
|
|
# keepalive between readings - matches the old 5s timer cadence
|
|
for _ in $(seq $((CHECK_INTERVAL / KEEPALIVE_INTERVAL))); do
|
|
sleep $KEEPALIVE_INTERVAL
|
|
set_fan "$SPEED"
|
|
done
|
|
done ;;
|
|
*) echo "usage: $0 [loop|once|temps|disks|selftest]" >&2; exit 1 ;;
|
|
esac
|