diff --git a/README.md b/README.md index 232f572..6d5d894 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,41 @@ hour, at which point the alarm is the outage. `smartctl -H` across two dozen drives runs hourly, not per loop. Absolute and trend checks stay on the fast loop. +## Calibrating a host + +**The defaults are not portable and the fans will be wrong on an uncalibrated host.** +Airflow, bay layout and the PWM→RPM relationship are all chassis specific. Calibration is +a one-off, takes about five minutes, and is what keeps this from being louder than stock: + +1. Hand the fans back to Dell and let them settle, then record what stock actually does: + `ipmitool raw 0x30 0x30 0x01 0x01`, wait a minute, then read + `ipmitool sdr type fan` and `ipmitool sdr type temperature`. +2. Take manual control (`ipmitool raw 0x30 0x30 0x01 0x00`) and sweep PWM to find the + percentage that reproduces that RPM: + `ipmitool raw 0x30 0x30 0x02 0xff 0x1c` (0x1c = 28%), waiting ~25s per step. +3. Set `LOW_FAN_SPEED` to that percentage. This is the floor the host idles at. +4. Set `CPU_RAMP_LOW_OFFSET` so the CPU ramp *starts a few degrees above the host's + normal idle*, so idle sits on the floor rather than permanently part-way up a ramp. + +Measured example — an R720xd (`iz-pve0`) whose CPUs idle at 60c and whose stock profile +holds 6240 RPM. Here 28% PWM == 6240 RPM, so `LOW_FAN_SPEED=28` and +`CPU_RAMP_LOW_OFFSET=15` (ramp starts at 62c). An R730xd (`iz-pve1`) idling at 48c is +fine on the 18/24 defaults. Put the values in the unit as `Environment=` lines. + +### Expect to run above stock on some hosts, legitimately + +Dell's profile does not look at drive temperature at all — only CPU, inlet and exhaust. +On `iz-pve0` two rear-bay SSDs sit in preheated exhaust air, and this was measured +directly: raising `DISK_RAMP_LOW_OFFSET` so those drives stopped voting dropped the fans +to 28% (exactly stock's 6240 RPM) and both SSDs climbed 54c → 61c within minutes. +Restoring their vote brought them back to 53-55c at 34% / 7080 RPM. + +So on that host **~13% more RPM than stock is the price of keeping two SSDs 6-8c cooler**, +and it is the whole reason this script exists. If you would rather have stock noise and +hotter drives, raise `DISK_RAMP_LOW_OFFSET` until the drives stop asking — that is a +preference, not a bug. What is *not* acceptable is being louder than stock for no reason, +which is what an uncalibrated CPU curve does. + ## Trend history Every pass appends to `log/temps.csv`: diff --git a/fan_speed.sh b/fan_speed.sh index 3793aab..97f58a3 100755 --- a/fan_speed.sh +++ b/fan_speed.sh @@ -27,15 +27,37 @@ IDRAC_HOST=local IDRAC_USERNAME=root IDRAC_PASSWORD=${IDRAC_PASSWORD:-calvin} CHECK_INTERVAL=10 #takes about 8 seconds to query all data -LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers -LOW_FAN_SPEED=18 #only decimal numbers -HIGH_FAN_SPEED=50 #only decimal numbers +# Per-chassis: the PWM that matches what Dell's own profile runs at idle. The +# PWM->RPM relationship is chassis specific, so this is measured, not guessed - +# see "Calibrating a host" in the README. 18/50 suits an R730xd that idles at +# 48c; an R720xd idling at 60c needs a floor of 28 to match stock. +LOW_FAN_SPEED=${LOW_FAN_SPEED:-18} +HIGH_FAN_SPEED=${HIGH_FAN_SPEED:-50} TABLE_HEADER_PRINT_INTERVAL=2 +# The CPU ramp is derived from the CPU sensor's own upper-non-critical +# threshold as reported by iDRAC, exactly like the disks are derived from +# theirs. A fixed 45c start was calibrated on a chassis that idles at 48c; on +# one that idles at 58c it sat 40% up the ramp doing nothing useful, and ran +# ~10% more airflow than Dell's own profile chose at the same temperature +# (6878 vs 6240 RPM measured over 8 minutes on an R720xd). +# +# Offsets chosen so a 58c idle lands near what Dell picks. Both an R720xd and +# an R730xd report upper-non-critical 77c, so this resolves to a 53->69c ramp. +CPU_RAMP_LOW_OFFSET=${CPU_RAMP_LOW_OFFSET:-24} +CPU_RAMP_HIGH_OFFSET=${CPU_RAMP_HIGH_OFFSET:-8} +CPU_LIMIT_FALLBACK=${CPU_LIMIT_FALLBACK:-77} + # Disk thresholds, alarm routing and trend window all live in monitor.sh. +# iDRAC's declared upper-non-critical temperature for the CPU sensor. +cpu_upper_non_critical() { + ipmitool -I $IDRAC_LOGIN_STRING sdr get "Temp" 2>/dev/null | + awk -F: '/Upper non-critical/ {gsub(/[^0-9.]/,"",$2); print int($2); exit}' +} + ####################################### # Self test - pure logic only, no IPMI and no drives touched. selftest() { @@ -83,15 +105,17 @@ selftest() { check "partial window stays silent" 0 "$(rise_over_window 4)" echo "fan speed deadband (hold unless the change is worth making):" - local now; now=$(date +%s) HIGH_FAN_SPEED=50 - should_apply_fan_speed 21 20 "$now" && check "1% drift holds" hold apply || check "1% drift holds" hold hold - should_apply_fan_speed 23 20 "$now" && check "3% drift holds" hold apply || check "3% drift holds" hold hold - should_apply_fan_speed 25 20 "$now" && check "5% rise applies" apply apply || check "5% rise applies" apply hold - should_apply_fan_speed 15 20 "$now" && check "5% fall applies" apply apply || check "5% fall applies" apply hold - should_apply_fan_speed 50 20 "$now" && check "full speed always applies" apply apply || check "full speed always applies" apply hold - should_apply_fan_speed 21 "" "$now" && check "first pass always applies" apply apply || check "first pass always applies" apply hold - should_apply_fan_speed 21 20 $((now - 400)) && check "stale setpoint is reasserted" apply apply || check "stale setpoint is reasserted" apply hold + check "1% drift holds" 20 "$(next_fan_speed 21 20)" + check "3% drift holds" 20 "$(next_fan_speed 23 20)" + check "4% drift holds" 32 "$(next_fan_speed 36 32)" + check "5% rise is adopted" 25 "$(next_fan_speed 25 20)" + check "5% fall is adopted" 15 "$(next_fan_speed 15 20)" + check "full speed always adopted" 50 "$(next_fan_speed 50 20)" + check "first pass adopts the request" 21 "$(next_fan_speed 21 '')" + # A re-assert must re-push what is held, not adopt the current want - that + # leak is what let the setpoint drift 32 -> 36 -> 31 on iz-pve0. + check "re-assert re-pushes held value" 32 "$(next_fan_speed 34 32)" echo "alert rate limiting:" local out @@ -146,14 +170,12 @@ fi ####################################### ####################################### -# This returns the lowest temp between CPU and GPU highest thresholds -# Used to calcuate fan speed interpolation -# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp -if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then - HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD -else - HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD -fi +# Derive the CPU ramp from what iDRAC says this CPU can actually take. +CPU_LIMIT=$(cpu_upper_non_critical) +[ -n "$CPU_LIMIT" ] || CPU_LIMIT=$CPU_LIMIT_FALLBACK +LOW_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_LOW_OFFSET)) +HIGH_TEMPERATURE_THRESHOLD=$((CPU_LIMIT - CPU_RAMP_HIGH_OFFSET)) +echo "CPU limit ${CPU_LIMIT}c -> fan ramp ${LOW_TEMPERATURE_THRESHOLD}c..${HIGH_TEMPERATURE_THRESHOLD}c" ####################################### ####################################### @@ -227,15 +249,16 @@ one_pass() { # CPU and GPU share one curve; disks are interpolated against their own limits # in retrieve_disk_temperatures. Each source asks for a speed, loudest wins - # temperatures from different classes of hardware are not comparable directly. - HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE - if [ "$GPU_TEMPERATURE" -gt "$HIGHEST_CPU_TEMPERATURE" ]; then - HIGHEST_TEMPERATURE=$GPU_TEMPERATURE - fi - if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then - CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_TEMPERATURE" \ - $LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED) - else - CPU_GPU_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED + CPU_GPU_FAN_SPEED=$(calculate_interpolated_fan_speed "$HIGHEST_CPU_TEMPERATURE" \ + $LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED) + # The GPU has its own limit and so gets its own ramp, rather than being + # compared against the CPU's on a shared scale. + if [ "$GPU_TEMPERATURE" -gt 0 ]; then + local gpu_speed + gpu_speed=$(calculate_interpolated_fan_speed "$GPU_TEMPERATURE" \ + $((GPU_TEMPERATURE_THRESHOLD - CPU_RAMP_LOW_OFFSET)) $GPU_TEMPERATURE_THRESHOLD \ + $LOW_FAN_SPEED $HIGH_FAN_SPEED) + [ "$gpu_speed" -gt "$CPU_GPU_FAN_SPEED" ] && CPU_GPU_FAN_SPEED=$gpu_speed fi DECIMAL_CURRENT_FAN_SPEED=$CPU_GPU_FAN_SPEED @@ -247,13 +270,15 @@ one_pass() { # Hold the current speed unless the change is worth making - see the deadband # note in monitor.sh. Chasing every degree is what made this louder than stock. - local held="" - if should_apply_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED" "$APPLIED_FAN_SPEED_AT"; then - apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED" - APPLIED_FAN_SPEED=$DECIMAL_CURRENT_FAN_SPEED - APPLIED_FAN_SPEED_AT=$(date +%s) - else - held=" held, want $DECIMAL_CURRENT_FAN_SPEED" + local target now held="" + target=$(next_fan_speed "$DECIMAL_CURRENT_FAN_SPEED" "$APPLIED_FAN_SPEED") + [ "$target" = "$DECIMAL_CURRENT_FAN_SPEED" ] || held=" held, want $DECIMAL_CURRENT_FAN_SPEED" + now=$(date +%s) + if [ "$target" != "$APPLIED_FAN_SPEED" ] || + [ $((now - APPLIED_FAN_SPEED_AT)) -ge "$FAN_REASSERT_INTERVAL" ]; then + apply_user_fan_control "$target" + APPLIED_FAN_SPEED=$target + APPLIED_FAN_SPEED_AT=$now fi COMMENT="CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Disk:$HOTTEST_DISK_TEMPERATURE/$HOTTEST_DISK_LIMIT($(basename $HOTTEST_DISK_DEVICE)) | Fan Speed:$APPLIED_FAN_SPEED($FAN_SPEED_DRIVER$held)" diff --git a/monitor.sh b/monitor.sh index e7e584c..b61a56e 100644 --- a/monitor.sh +++ b/monitor.sh @@ -52,20 +52,30 @@ FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-5} # ipmitool call every 5 minutes is cheaper than finding out the hard way. FAN_REASSERT_INTERVAL=${FAN_REASSERT_INTERVAL:-300} -# should_apply_fan_speed -# True when the change is worth making. A request for full speed is never held -# back, and the first pass always applies. -should_apply_fan_speed() { - local wanted=$1 applied=$2 applied_at=$3 delta now - [ -n "$applied" ] || return 0 +# fan_speed_changed_enough +# True when the request has moved far enough to be worth acting on. A request +# for full speed is never held back. +fan_speed_changed_enough() { + local wanted=$1 applied=$2 delta [ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0 - now=$(date +%s) - [ $((now - applied_at)) -ge "$FAN_REASSERT_INTERVAL" ] && return 0 delta=$((wanted - applied)) [ "$delta" -lt 0 ] && delta=$((-delta)) [ "$delta" -ge "$FAN_SPEED_DEADBAND" ] } +# next_fan_speed -> the speed to hold from here. +# Kept separate from the re-assert timer on purpose: a re-assert must re-push +# the value already being held, never silently adopt the current request, or +# the deadband leaks a few percent every FAN_REASSERT_INTERVAL. +next_fan_speed() { + local wanted=$1 applied=$2 + if [ -z "$applied" ] || fan_speed_changed_enough "$wanted" "$applied"; then + echo "$wanted" + else + echo "$applied" + fi +} + ALERT_EMAIL=${ALERT_EMAIL:-Servers@ntfy1.izebra.xyz} ALERT_COOLDOWN=${ALERT_COOLDOWN:-3600} STATE_DIR=${STATE_DIR:-/root/fan_speed/state}