fan_speed/monitor.sh

#!/bin/bash
#
# Disk temperature monitoring, trend tracking and email alarms.
#
# Sourced by fan_speed.sh alongside functions.sh. functions.sh is vendored from
# upstream (tigerblue77/Dell_iDRAC_fan_controller_Docker); everything in here is
# local, so keep the two files separate to keep that boundary readable.

# ---------------------------------------------------------------- configuration

DISK_GLOB=${DISK_GLOB:-/dev/disk/by-path/*}

# Drives to leave alone, matched against the by-path link. Set this to the PCI
# address of an external shelf's HBA on any host that has one.
#
# There is no reliable way to infer this. An external enclosure looks exactly
# like an internal backplane from /dev/disk/by-path: iz-pve1's MD1200 sits
# behind its own HBA at pci-0000:04:00.0, while iz-pve0's *internal* drives sit
# behind a SAS expander at pci-0000:02:00.0. Guessing from "-sas-exp" picks up
# the wrong set on one host or the other, so it has to be stated per host.
DISK_EXCLUDE_PATTERN=${DISK_EXCLUDE_PATTERN:-}

# Each drive's fan ramp is derived from its own maximum operating temperature:
#   ramp starts at  limit - DISK_RAMP_LOW_OFFSET
#   full speed at   limit - DISK_RAMP_HIGH_OFFSET
# Larger low offset = react earlier = louder. See README before changing.
DISK_RAMP_LOW_OFFSET=${DISK_RAMP_LOW_OFFSET:-18}
DISK_RAMP_HIGH_OFFSET=${DISK_RAMP_HIGH_OFFSET:-8}

# A drive this close to its own limit raises an alarm - the fans are already
# flat out for it and it is still climbing.
#
# That sentence is only true if full fan speed is reached at or before the alarm
# point, i.e. DISK_RAMP_HIGH_OFFSET >= DISK_ALARM_OFFSET. Set the other way round
# the alarm mails you while there is still cooling left unused, which is exactly
# backwards. check_ramp_ordering() enforces it at startup.
DISK_ALARM_OFFSET=${DISK_ALARM_OFFSET:-5}

# True when full cooling is reached no later than the alarm point.
ramp_ordering_ok() {
  [ "${1:-$DISK_RAMP_HIGH_OFFSET}" -ge "${2:-$DISK_ALARM_OFFSET}" ]
}

# Clamp rather than refuse to start - a fan controller that exits leaves the
# fans wherever they were, which is worse than a slightly wrong curve.
check_ramp_ordering() {
  if ! ramp_ordering_ok; then
    echo "/!\\ DISK_RAMP_HIGH_OFFSET ($DISK_RAMP_HIGH_OFFSET) is below DISK_ALARM_OFFSET ($DISK_ALARM_OFFSET):" >&2
    echo "    the alarm would fire before the fans reach full speed. Raising it to $DISK_ALARM_OFFSET." >&2
    log_line "config: raised DISK_RAMP_HIGH_OFFSET $DISK_RAMP_HIGH_OFFSET -> $DISK_ALARM_OFFSET"
    DISK_RAMP_HIGH_OFFSET=$DISK_ALARM_OFFSET
  fi
}

# Drives report their limit but do not agree on what it means: Samsung and
# Kioxia report a true operating maximum (70), Toshiba and Seagate report 60,
# WD Reds report 85 - the SCT critical limit, not an operating maximum.
# Clamp by class so one optimistic drive cannot quietly disable cooling.
HDD_LIMIT_CAP=${HDD_LIMIT_CAP:-60}
SSD_LIMIT_CAP=${SSD_LIMIT_CAP:-70}

# The fan speed is only pushed to the BMC when it moves by at least this much.
#
# Without it the setpoint is a continuous function of instantaneous temperature:
# the CPU curve is ~1% of fan per degree, idle CPU noise is +/-2c, so the fan
# gets a new speed every single pass and never settles. Measured on iz-pve0,
# stock iDRAC held one speed for 5 minutes straight through the same jitter
# while this script made 7 changes in 16 minutes.
FAN_SPEED_DEADBAND=${FAN_SPEED_DEADBAND:-5}

# Re-push the held speed this often anyway, in case the BMC forgets it. Not
# measured - the MD1200 EMM does forget, iDRAC is believed not to, and one
# ipmitool call every 5 minutes is cheaper than finding out the hard way.
FAN_REASSERT_INTERVAL=${FAN_REASSERT_INTERVAL:-300}

# fan_speed_changed_enough <wanted> <currently applied>
# True when the request has moved far enough to be worth acting on. A request
# for full speed is never held back.
fan_speed_changed_enough() {
  local wanted=$1 applied=$2 delta
  [ "$wanted" -ge "$HIGH_FAN_SPEED" ] && return 0
  delta=$((wanted - applied))
  [ "$delta" -lt 0 ] && delta=$((-delta))
  [ "$delta" -ge "$FAN_SPEED_DEADBAND" ]
}

# next_fan_speed <wanted> <currently applied> -> the speed to hold from here.
# Kept separate from the re-assert timer on purpose: a re-assert must re-push
# the value already being held, never silently adopt the current request, or
# the deadband leaks a few percent every FAN_REASSERT_INTERVAL.
next_fan_speed() {
  local wanted=$1 applied=$2
  if [ -z "$applied" ] || fan_speed_changed_enough "$wanted" "$applied"; then
    echo "$wanted"
  else
    echo "$applied"
  fi
}

ALERT_EMAIL=${ALERT_EMAIL:-Servers@ntfy1.izebra.xyz}
ALERT_COOLDOWN=${ALERT_COOLDOWN:-3600}
STATE_DIR=${STATE_DIR:-/root/fan_speed/state}
LOG_FILE=${LOG_FILE:-/root/fan_speed/log/fan_speed.log}

TREND_FILE=${TREND_FILE:-/root/fan_speed/log/temps.csv}
TREND_SAMPLES=${TREND_SAMPLES:-90}       # 90 x 10s CHECK_INTERVAL = 15 minutes
TREND_RISE_ALARM=${TREND_RISE_ALARM:-8}  # degrees of climb across that window

SMART_CHECK_INTERVAL=${SMART_CHECK_INTERVAL:-3600}

# ------------------------------------------------------------------ disk temps

log_line() {
  echo "$(date +'%Y-%m-%d %H:%M:%S') || $*" >> "$LOG_FILE"
}

# Block devices behind DISK_GLOB, deduplicated. Resolved fresh every call:
# kernel names are not stable, a shelf rescan renamed sdaa-sdai to sds-sdaa.
disk_devices() {
  local link dev
  for link in $DISK_GLOB; do
    case "$link" in *-part*) continue ;; esac
    if [ -n "$DISK_EXCLUDE_PATTERN" ]; then
      case "$link" in *"$DISK_EXCLUDE_PATTERN"*) continue ;; esac
    fi
    [ -e "$link" ] || continue
    dev=$(readlink -f "$link")
    [ -b "$dev" ] && echo "$dev"
  done | sort -u
}

# smartctl -A output on stdin -> temperature in celsius, empty if it did not say.
# Drives disagree on the attribute name: 194 Temperature_Celsius (most),
# 190 Airflow_Temperature_Cel (Seagate), 194 Temperature_Internal (Intel),
# and SCSI/SAS drives report a "Current Drive Temperature" line instead.
# Temperature_Difference_from_100 is a delta, not a reading - never match it.
parse_disk_temperature() {
  awk '
    /Current Drive Temperature/           { print $4; exit }
    /^ *[0-9]+ +[A-Za-z_]*Temperature[A-Za-z_]*/ && !/Difference/ { print $10; exit }'
}

# smartctl -x output on stdin -> the drive's own maximum operating temperature.
parse_disk_limit() {
  awk '
    /Min\/Max Temperature Limit/ {
      if (match($0, /-?[0-9]+\/[0-9]+/)) { split(substr($0, RSTART, RLENGTH), a, "/"); print a[2] }
      exit
    }
    /Drive Trip Temperature/ { print $4; exit }'
}

# -n standby: skip a sleeping drive rather than spinning it up to measure it.
disk_temperature() {
  smartctl -n standby -A "$1" 2>/dev/null | parse_disk_temperature
}

# The drive's own limit, clamped by class (see HDD_LIMIT_CAP above).
disk_limit() {
  local dev=$1 limit rotational cap
  limit=$(smartctl -x "$dev" 2>/dev/null | parse_disk_limit)
  rotational=$(cat "/sys/block/$(basename "$dev")/queue/rotational" 2>/dev/null)
  if [ "$rotational" = 1 ]; then cap=$HDD_LIMIT_CAP; else cap=$SSD_LIMIT_CAP; fi
  if [ -z "$limit" ] || ! [ "$limit" -le "$cap" ] 2>/dev/null; then limit=$cap; fi
  echo "$limit"
}

# Limits do not change, and smartctl -x is far heavier than -A. Read them once.
declare -A DISK_LIMIT
cache_disk_limits() {
  local dev
  DISK_LIMIT=()
  for dev in $(disk_devices); do
    DISK_LIMIT[$dev]=$(disk_limit "$dev")
  done
  DISK_COUNT_EXPECTED=${#DISK_LIMIT[@]}
}

# Reads every drive. Sets:
#   HOTTEST_DISK_TEMPERATURE / _DEVICE / _LIMIT, DISK_FAN_SPEED, DISKS_READ
# Each drive is interpolated against its OWN limit and the highest resulting fan
# speed wins - so a 50c SSD rated to 70 asks for less than a 45c disk rated to 60.
# That makes the curve independent of drive technology and of bay position.
retrieve_disk_temperatures() {
  local dev temp limit speed
  HOTTEST_DISK_TEMPERATURE=0
  HOTTEST_DISK_DEVICE="-"
  HOTTEST_DISK_LIMIT=0
  DISK_FAN_SPEED=$LOW_FAN_SPEED
  DISKS_READ=0

  for dev in $(disk_devices); do
    temp=$(disk_temperature "$dev")
    [ -n "$temp" ] || continue
    DISKS_READ=$((DISKS_READ + 1))

    limit=${DISK_LIMIT[$dev]}
    [ -n "$limit" ] || limit=$(disk_limit "$dev")

    speed=$(calculate_interpolated_fan_speed "$temp" \
              $((limit - DISK_RAMP_LOW_OFFSET)) $((limit - DISK_RAMP_HIGH_OFFSET)) \
              "$LOW_FAN_SPEED" "$HIGH_FAN_SPEED")
    [ "$speed" -gt "$DISK_FAN_SPEED" ] && DISK_FAN_SPEED=$speed

    if [ "$temp" -gt "$HOTTEST_DISK_TEMPERATURE" ]; then
      HOTTEST_DISK_TEMPERATURE=$temp
      HOTTEST_DISK_DEVICE=$dev
      HOTTEST_DISK_LIMIT=$limit
    fi
  done
}

# --------------------------------------------------------------------- alarms

notify() {
  local subject=$1 body=$2
  if [ -n "$DRY_RUN" ]; then
    echo "MAIL[$ALERT_EMAIL] $subject"
  else
    printf '%s\n' "$body" | mail -s "$subject" "$ALERT_EMAIL"
  fi
  log_line "ALERT $subject"
}

# raise_alert <key> <subject> <body>
# Rate limited per key. Without this a 10s loop sends 360 mails an hour and the
# alarm becomes the outage.
raise_alert() {
  local key=$1 subject=$2 body=$3
  local stamp="$STATE_DIR/$key.alert" now last
  mkdir -p "$STATE_DIR"
  now=$(date +%s)
  if [ -f "$stamp" ]; then
    last=$(cat "$stamp")
    [ $((now - last)) -lt "$ALERT_COOLDOWN" ] && return 0
  fi
  echo "$now" > "$stamp"
  notify "$subject" "$body"
}

# clear_alert <key> <subject> - one recovery notice, only if the alarm was up.
clear_alert() {
  local key=$1 subject=$2
  local stamp="$STATE_DIR/$key.alert"
  [ -f "$stamp" ] || return 0
  rm -f "$stamp"
  notify "$subject" "Recovered at $(date '+%Y-%m-%d %H:%M:%S')."
}

# ---------------------------------------------------------------------- trend

# record_sample <cpu> <gpu> <disk> <fan>
record_sample() {
  mkdir -p "$(dirname "$TREND_FILE")"
  echo "$(date +%s),$1,$2,$3,$4" >> "$TREND_FILE"
  if [ "$(wc -l < "$TREND_FILE")" -gt $((TREND_SAMPLES * 2)) ]; then
    tail -n "$TREND_SAMPLES" "$TREND_FILE" > "$TREND_FILE.tmp" && mv "$TREND_FILE.tmp" "$TREND_FILE"
  fi
}

# Is the cooling losing? A rising temperature on its own is not a fault - it is
# what a busy machine looks like, and a zpool migration will do it for hours.
# What matters is whether the fans are answering it:
#
#   no_headroom    temperature climbing while the fans are already flat out.
#                  Unambiguous: there is nothing left to give.
#   not_converging temperature climbing, fans climbing with it, and the rise is
#                  not slowing down. Cooling is responding but losing ground.
#   quiet          anything else, including a big climb that the fans absorbed
#                  and that has since plateaued. That is just load.
#
# Compares the first half of the window against the second to tell a plateau
# from a runaway - a normal load step decelerates, a cooling failure does not.
#
# trend_verdict <temperature column> <fan column>
trend_verdict() {
  local tcol=$1 fcol=$2 lines window oldest mid newest fan_old fan_new first second
  lines=$(wc -l < "$TREND_FILE" 2>/dev/null || echo 0)
  if [ "$lines" -lt "$TREND_SAMPLES" ]; then echo quiet; return; fi
  window=$(tail -n "$TREND_SAMPLES" "$TREND_FILE")

  oldest=$(printf '%s\n' "$window" | head -1        | cut -d, -f"$tcol")
  mid=$(printf   '%s\n' "$window" | sed -n "$(( (TREND_SAMPLES + 1) / 2 ))p" | cut -d, -f"$tcol")
  newest=$(printf '%s\n' "$window" | tail -1        | cut -d, -f"$tcol")
  fan_old=$(printf '%s\n' "$window" | head -1       | cut -d, -f"$fcol")
  fan_new=$(printf '%s\n' "$window" | tail -1       | cut -d, -f"$fcol")
  [ -n "$oldest" ] && [ -n "$mid" ] && [ -n "$newest" ] || { echo quiet; return; }

  # Not climbing meaningfully - nothing to say either way.
  [ $((newest - oldest)) -ge "$TREND_RISE_ALARM" ] || { echo quiet; return; }

  # Climbing with the fans already flat out.
  if [ "$fan_new" -ge "$HIGH_FAN_SPEED" ]; then echo no_headroom; return; fi

  # Climbing, fans climbing too, and the second half of the window rose at least
  # as fast as the first - it is not settling.
  first=$((mid - oldest))
  second=$((newest - mid))
  if [ "$fan_new" -gt "$fan_old" ] && [ "$second" -ge "$first" ]; then
    echo not_converging; return
  fi

  echo quiet
}

# rise_over_window <column> -> degrees climbed from the oldest sample in the
# window to the newest. 0 until there is a full window of history, so a restart
# cannot alarm on a partial series. Reported in alarm text; the decision to
# alarm belongs to trend_verdict.
# Columns: 2 cpu, 3 gpu, 4 disk, 5 fan.
rise_over_window() {
  local column=$1 oldest newest lines
  lines=$(wc -l < "$TREND_FILE" 2>/dev/null || echo 0)
  if [ "$lines" -lt "$TREND_SAMPLES" ]; then echo 0; return; fi
  oldest=$(tail -n "$TREND_SAMPLES" "$TREND_FILE" | head -1 | cut -d, -f"$column")
  newest=$(tail -n 1 "$TREND_FILE" | cut -d, -f"$column")
  [ -n "$oldest" ] && [ -n "$newest" ] || { echo 0; return; }
  echo $((newest - oldest))
}

# --------------------------------------------------------------- SMART health

# Expensive across two dozen drives - the caller runs this hourly, not per loop.
check_disk_health() {
  local dev status failed=""
  for dev in $(disk_devices); do
    status=$(smartctl -H "$dev" 2>/dev/null | grep -iE "overall-health|SMART Health Status")
    [ -n "$status" ] || continue
    case "$status" in
      *PASSED*|*OK*) ;;
      *) failed+="$dev: $status"$'\n' ;;
    esac
  done
  if [ -n "$failed" ]; then
    raise_alert smart "SMART failure on $(hostname)" "$failed"
  else
    clear_alert smart "SMART recovered on $(hostname)"
  fi
}