Add temperature-aware MD1200 fan control
das_fanctl.sh reads drive temps via smartctl (falling back to the EMM backplane sensors), interpolates a fan percentage, and re-sends _shutup on a 5s keepalive so the shelf does not revert to full speed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
191
fan_speed.sh
Executable file
191
fan_speed.sh
Executable file
@@ -0,0 +1,191 @@
|
||||
#!/bin/bash
|
||||
|
||||
source /root/fan_speed/functions.sh
|
||||
|
||||
# Trap the signals for container exit and run graceful_exit function
|
||||
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
|
||||
|
||||
# Variable defintions
|
||||
# LOW_FAN_SPEED = Lowest fan speed
|
||||
# HIGH_FAN_SPEED = max fan speed. (0-100)
|
||||
# IDRAC_HOST = local or IP of iDRAC
|
||||
# IDRAC_USERNAME = username of iDRAC
|
||||
# IDRAC_PASSWORD = password if iDRAC
|
||||
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
|
||||
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
|
||||
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
|
||||
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
|
||||
|
||||
|
||||
IDRAC_HOST=local
|
||||
IDRAC_USERNAME=root
|
||||
IDRAC_PASSWORD=calvin
|
||||
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
|
||||
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
|
||||
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
|
||||
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
|
||||
LOW_FAN_SPEED=18 #only decimal numbers
|
||||
HIGH_FAN_SPEED=50 #only decimal numbers
|
||||
TABLE_HEADER_PRINT_INTERVAL=2
|
||||
|
||||
|
||||
#######################################
|
||||
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
|
||||
if [[ $IDRAC_HOST == "local" ]]; then
|
||||
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
|
||||
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
|
||||
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
IDRAC_LOGIN_STRING='open'
|
||||
else
|
||||
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
|
||||
echo "iDRAC/IPMI password: $IDRAC_PASSWORD"
|
||||
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# This script ONLY runs on Dell Servers
|
||||
get_Dell_server_model
|
||||
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
|
||||
echo "/!\ Your server isn't a Dell product. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
#****Probably don't need this any more****
|
||||
# Prepare, format and define initial variables
|
||||
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
|
||||
if [[ $LOW_FAN_SPEED == 0x* ]]; then
|
||||
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
|
||||
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
||||
else
|
||||
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
||||
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# This returns the lowest temp between CPU and GPU highest thresholds
|
||||
# Used to calcuate fan speed interpolation
|
||||
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
|
||||
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
|
||||
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
|
||||
else
|
||||
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Determine where to get CPU temps from iDRAC
|
||||
# If server model is Gen 14 (*40) or newer
|
||||
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
|
||||
DELL_POWEREDGE_GEN_14_OR_NEWER=true
|
||||
CPU1_TEMPERATURE_INDEX=2
|
||||
CPU2_TEMPERATURE_INDEX=4
|
||||
else
|
||||
DELL_POWEREDGE_GEN_14_OR_NEWER=false
|
||||
CPU1_TEMPERATURE_INDEX=1
|
||||
CPU2_TEMPERATURE_INDEX=2
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Check if sensors are present on the server
|
||||
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
|
||||
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
|
||||
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
if [ -z "$EXHAUST_TEMPERATURE" ]; then
|
||||
echo "No exhaust temperature sensor detected."
|
||||
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
|
||||
fi
|
||||
if [ -z "$CPU2_TEMPERATURE" ]; then
|
||||
echo "No CPU2 temperature sensor detected."
|
||||
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
|
||||
fi
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Print out config information before starting
|
||||
#echo "Server Manufacturer: $SERVER_MANUFACTURER"
|
||||
#echo "Server Model: $SERVER_MODEL"
|
||||
#echo "iDRAC Host: $IDRAC_HOST"
|
||||
#echo "Check Interval: $CHECK_INTERVAL"
|
||||
#echo "Low Temp Threshold: $LOW_TEMPERATURE_THRESHOLD"
|
||||
#echo "CPU High Temp threshold: $CPU_TEMPERATURE_THRESHOLD"
|
||||
#echo "GPU High Temp Threshold: $GPU_TEMPERATURE_THRESHOLD"
|
||||
#echo "Low Fan Speed: $LOW_FAN_SPEED"
|
||||
#echo "High Fan Speed: $HIGH_FAN_SPEED"
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# Loop (this is where the magic happens)
|
||||
while true; do
|
||||
# Sleep for the specified interval before taking another reading
|
||||
sleep $CHECK_INTERVAL &
|
||||
SLEEP_PROCESS_PID=$!
|
||||
|
||||
#Get Temps
|
||||
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
retrieve_gpu_temperature
|
||||
|
||||
i=0
|
||||
# Get highest CPU Temp and check if any of them are over heating
|
||||
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
|
||||
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
||||
if [ $CPU2_TEMPERATURE -gt $CPU1_TEMPERATURE ]; then
|
||||
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
|
||||
fi
|
||||
fi
|
||||
|
||||
#echo $HIGHEST_CPU_TEMPERATURE
|
||||
|
||||
|
||||
# Define Function to check for if CPU temps are greater than the threshold
|
||||
function CPU_OVERHEAT() { [ $HIGHEST_CPU_TEMPERATURE -gt $CPU_TEMPERATURE_THRESHOLD ]; }
|
||||
|
||||
# Define function to check if GPU temperature are above the threshold
|
||||
function GPU_OVERHEAT() { [ $GPU_TEMPERATURE -gt $GPU_TEMPERATURE_THRESHOLD ]; }
|
||||
|
||||
# Check if CPU is overheating then apply Dell default dynamic fan control profile if true
|
||||
# or set CPU fan based on interpolated temps
|
||||
if CPU_OVERHEAT; then
|
||||
apply_Dell_fan_control_profile
|
||||
if ! $IS_DELL_FAN_CONTROL_PROFILE_APPLIED; then
|
||||
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
|
||||
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
|
||||
fi
|
||||
else
|
||||
#this combines CPU and GPU temps and outputs which one is highest
|
||||
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
|
||||
if [ $GPU_TEMPERATURE -gt $HIGHEST_CPU_TEMPERATURE ]; then
|
||||
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
|
||||
fi
|
||||
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
|
||||
DECIMAL_CURRENT_FAN_SPEED=$(calculate_interpolated_fan_speed $HIGHEST_TEMPERATURE $LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
||||
else
|
||||
DECIMAL_CURRENT_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
|
||||
fi
|
||||
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
|
||||
COMMENT=$(echo "CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED")
|
||||
fi
|
||||
|
||||
if GPU_OVERHEAT; then
|
||||
COMMENT="GPU temperature is too high, killing CT(s) to prevent GPU damage"
|
||||
# This will eventually have a function to kill the CT(s) that use the Tesla P4
|
||||
fi
|
||||
|
||||
# Print temperatures, active fan control profile and comment if any change happened during last time interval
|
||||
if [ $i -eq $TABLE_HEADER_PRINT_INTERVAL ]; then
|
||||
#echo " ------- Temperatures -------"
|
||||
#$echo " Date & time Inlet CPU 1 CPU 2 Exhaust Active fan speed profile Third-party PCIe card Dell default cooling response Comment"
|
||||
i=0
|
||||
fi
|
||||
#printf "%19s %3d°C %3d°C %3s°C %5s°C %40s %51s %s\n" "$(date +"%d-%m-%Y %T")" $INLET_TEMPERATURE $CPU1_TEMPERATURE "$CPU2_TEMPERATURE" "$EXHAUST_TEMPERATURE" "$CURRENT_FAN_CONTROL_PROFILE" "$THIRD_PARTY_PCIE_CARD_DELL_DEFAULT_COOLING_RESPONSE_STATUS" "$COMMENT"
|
||||
z_date=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
echo "$z_date || $COMMENT" >> /root/fan_speed/log/fan_speed.log
|
||||
((i++))
|
||||
wait $SLEEP_PROCESS_PID
|
||||
done
|
||||
Reference in New Issue
Block a user