db69c08c73
das_fanctl.sh reads drive temps via smartctl (falling back to the EMM backplane sensors), interpolates a fan percentage, and re-sends _shutup on a 5s keepalive so the shelf does not revert to full speed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
192 lines
7.9 KiB
Bash
Executable File
192 lines
7.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
source /root/fan_speed/functions.sh
|
|
|
|
# Trap the signals for container exit and run graceful_exit function
|
|
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM
|
|
|
|
# Variable defintions
|
|
# LOW_FAN_SPEED = Lowest fan speed
|
|
# HIGH_FAN_SPEED = max fan speed. (0-100)
|
|
# IDRAC_HOST = local or IP of iDRAC
|
|
# IDRAC_USERNAME = username of iDRAC
|
|
# IDRAC_PASSWORD = password if iDRAC
|
|
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
|
|
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
|
|
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
|
|
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps
|
|
|
|
|
|
IDRAC_HOST=local
|
|
IDRAC_USERNAME=root
|
|
IDRAC_PASSWORD=calvin
|
|
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
|
|
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
|
|
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
|
|
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
|
|
LOW_FAN_SPEED=18 #only decimal numbers
|
|
HIGH_FAN_SPEED=50 #only decimal numbers
|
|
TABLE_HEADER_PRINT_INTERVAL=2
|
|
|
|
|
|
#######################################
|
|
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
|
|
if [[ $IDRAC_HOST == "local" ]]; then
|
|
# Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
|
|
if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
|
|
echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
|
|
exit 1
|
|
fi
|
|
IDRAC_LOGIN_STRING='open'
|
|
else
|
|
echo "iDRAC/IPMI username: $IDRAC_USERNAME"
|
|
echo "iDRAC/IPMI password: $IDRAC_PASSWORD"
|
|
IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# This script ONLY runs on Dell Servers
|
|
get_Dell_server_model
|
|
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
|
|
echo "/!\ Your server isn't a Dell product. Exiting." >&2
|
|
exit 1
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
#****Probably don't need this any more****
|
|
# Prepare, format and define initial variables
|
|
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
|
|
if [[ $LOW_FAN_SPEED == 0x* ]]; then
|
|
readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
|
|
readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
|
else
|
|
readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
|
|
readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# This returns the lowest temp between CPU and GPU highest thresholds
|
|
# Used to calcuate fan speed interpolation
|
|
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
|
|
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
|
|
HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
|
|
else
|
|
HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Determine where to get CPU temps from iDRAC
|
|
# If server model is Gen 14 (*40) or newer
|
|
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
|
|
DELL_POWEREDGE_GEN_14_OR_NEWER=true
|
|
CPU1_TEMPERATURE_INDEX=2
|
|
CPU2_TEMPERATURE_INDEX=4
|
|
else
|
|
DELL_POWEREDGE_GEN_14_OR_NEWER=false
|
|
CPU1_TEMPERATURE_INDEX=1
|
|
CPU2_TEMPERATURE_INDEX=2
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Check if sensors are present on the server
|
|
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
|
|
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
|
|
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
|
if [ -z "$EXHAUST_TEMPERATURE" ]; then
|
|
echo "No exhaust temperature sensor detected."
|
|
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
|
|
fi
|
|
if [ -z "$CPU2_TEMPERATURE" ]; then
|
|
echo "No CPU2 temperature sensor detected."
|
|
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
|
|
fi
|
|
#######################################
|
|
|
|
#######################################
|
|
# Print out config information before starting
|
|
#echo "Server Manufacturer: $SERVER_MANUFACTURER"
|
|
#echo "Server Model: $SERVER_MODEL"
|
|
#echo "iDRAC Host: $IDRAC_HOST"
|
|
#echo "Check Interval: $CHECK_INTERVAL"
|
|
#echo "Low Temp Threshold: $LOW_TEMPERATURE_THRESHOLD"
|
|
#echo "CPU High Temp threshold: $CPU_TEMPERATURE_THRESHOLD"
|
|
#echo "GPU High Temp Threshold: $GPU_TEMPERATURE_THRESHOLD"
|
|
#echo "Low Fan Speed: $LOW_FAN_SPEED"
|
|
#echo "High Fan Speed: $HIGH_FAN_SPEED"
|
|
#######################################
|
|
|
|
#######################################
|
|
# Loop (this is where the magic happens)
|
|
while true; do
|
|
# Sleep for the specified interval before taking another reading
|
|
sleep $CHECK_INTERVAL &
|
|
SLEEP_PROCESS_PID=$!
|
|
|
|
#Get Temps
|
|
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
|
retrieve_gpu_temperature
|
|
|
|
i=0
|
|
# Get highest CPU Temp and check if any of them are over heating
|
|
HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
|
|
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
|
if [ $CPU2_TEMPERATURE -gt $CPU1_TEMPERATURE ]; then
|
|
HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
|
|
fi
|
|
fi
|
|
|
|
#echo $HIGHEST_CPU_TEMPERATURE
|
|
|
|
|
|
# Define Function to check for if CPU temps are greater than the threshold
|
|
function CPU_OVERHEAT() { [ $HIGHEST_CPU_TEMPERATURE -gt $CPU_TEMPERATURE_THRESHOLD ]; }
|
|
|
|
# Define function to check if GPU temperature are above the threshold
|
|
function GPU_OVERHEAT() { [ $GPU_TEMPERATURE -gt $GPU_TEMPERATURE_THRESHOLD ]; }
|
|
|
|
# Check if CPU is overheating then apply Dell default dynamic fan control profile if true
|
|
# or set CPU fan based on interpolated temps
|
|
if CPU_OVERHEAT; then
|
|
apply_Dell_fan_control_profile
|
|
if ! $IS_DELL_FAN_CONTROL_PROFILE_APPLIED; then
|
|
IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
|
|
COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
|
|
fi
|
|
else
|
|
#this combines CPU and GPU temps and outputs which one is highest
|
|
HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
|
|
if [ $GPU_TEMPERATURE -gt $HIGHEST_CPU_TEMPERATURE ]; then
|
|
HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
|
|
fi
|
|
if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
|
|
DECIMAL_CURRENT_FAN_SPEED=$(calculate_interpolated_fan_speed $HIGHEST_TEMPERATURE $LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
|
|
else
|
|
DECIMAL_CURRENT_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
|
|
fi
|
|
apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
|
|
COMMENT=$(echo "CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED")
|
|
fi
|
|
|
|
if GPU_OVERHEAT; then
|
|
COMMENT="GPU temperature is too high, killing CT(s) to prevent GPU damage"
|
|
# This will eventually have a function to kill the CT(s) that use the Tesla P4
|
|
fi
|
|
|
|
# Print temperatures, active fan control profile and comment if any change happened during last time interval
|
|
if [ $i -eq $TABLE_HEADER_PRINT_INTERVAL ]; then
|
|
#echo " ------- Temperatures -------"
|
|
#$echo " Date & time Inlet CPU 1 CPU 2 Exhaust Active fan speed profile Third-party PCIe card Dell default cooling response Comment"
|
|
i=0
|
|
fi
|
|
#printf "%19s %3d°C %3d°C %3s°C %5s°C %40s %51s %s\n" "$(date +"%d-%m-%Y %T")" $INLET_TEMPERATURE $CPU1_TEMPERATURE "$CPU2_TEMPERATURE" "$EXHAUST_TEMPERATURE" "$CURRENT_FAN_CONTROL_PROFILE" "$THIRD_PARTY_PCIE_CARD_DELL_DEFAULT_COOLING_RESPONSE_STATUS" "$COMMENT"
|
|
z_date=$(date +"%Y-%m-%d %H:%M:%S")
|
|
echo "$z_date || $COMMENT" >> /root/fan_speed/log/fan_speed.log
|
|
((i++))
|
|
wait $SLEEP_PROCESS_PID
|
|
done
|