Add temperature-aware MD1200 fan control
das_fanctl.sh reads drive temps via smartctl (falling back to the EMM backplane sensors), interpolates a fan percentage, and re-sends _shutup on a 5s keepalive so the shelf does not revert to full speed. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
215
functions.sh
Executable file
215
functions.sh
Executable file
@@ -0,0 +1,215 @@
|
||||
# Define global functions
|
||||
# This function applies Dell's default dynamic fan control profile
|
||||
function apply_Dell_fan_control_profile() {
|
||||
# Use ipmitool to send the raw command to set fan control to Dell default
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x01 0x01 > /dev/null
|
||||
CURRENT_FAN_CONTROL_PROFILE="Dell default dynamic fan control profile"
|
||||
}
|
||||
|
||||
#Get the TEMP of the TESLA P4 installed in this servers
|
||||
#Should probably set a variable so this function can check temps of any GPU using nvidia-smi
|
||||
retrieve_gpu_temperature() {
|
||||
if [ -x /usr/bin/nvidia-smi ]; then
|
||||
# Get the index of the Tesla P4 GPU using nvidia-smi
|
||||
index=$(nvidia-smi --query-gpu=name,index --format=csv | grep "Tesla P4" | cut -d ',' -f 2)
|
||||
else
|
||||
index=""
|
||||
fi
|
||||
|
||||
if [ "$index" != "" ]; then
|
||||
# Get the temperature of the Tesla P4 GPU using nvidia-smi and the index obtained above
|
||||
GPU_TEMPERATURE=$(nvidia-smi --query-gpu=temperature.gpu --format=csv | head -$(( $(($index + 2)) )) | tail -1)
|
||||
else
|
||||
GPU_TEMPERATURE="0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Apply user-defined fan control settings
|
||||
#
|
||||
# This function applies user-defined fan control settings based on the fan speed.
|
||||
# It handles both decimal and hexadecimal fan speed inputs, converting between them as needed.
|
||||
# The function then applies the fan control and updates the current fan control profile.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (LOCAL_FAN_SPEED): The desired fan speed. Can be in decimal (0-100) or hexadecimal (0x00-0x64) format.
|
||||
#
|
||||
# Global variables used:
|
||||
# CURRENT_FAN_CONTROL_PROFILE: Updated with the current fan control profile description.
|
||||
#
|
||||
# Returns:
|
||||
# None.
|
||||
function apply_user_fan_control() {
|
||||
local LOCAL_FAN_SPEED=$1
|
||||
|
||||
if [[ $LOCAL_FAN_SPEED == 0x* ]]; then
|
||||
local LOCAL_DECIMAL_FAN_SPEED
|
||||
LOCAL_DECIMAL_FAN_SPEED=$(printf '%d' "$LOCAL_FAN_SPEED")
|
||||
local LOCAL_HEXADECIMAL_FAN_SPEED=$LOCAL_FAN_SPEED
|
||||
else
|
||||
local LOCAL_DECIMAL_FAN_SPEED=$LOCAL_FAN_SPEED
|
||||
local LOCAL_HEXADECIMAL_FAN_SPEED
|
||||
LOCAL_HEXADECIMAL_FAN_SPEED=$(convert_decimal_value_to_hexadecimal "$LOCAL_FAN_SPEED")
|
||||
fi
|
||||
|
||||
apply_fan_control_to_specified_value "$LOCAL_HEXADECIMAL_FAN_SPEED"
|
||||
CURRENT_FAN_CONTROL_PROFILE="User static fan control profile ($LOCAL_DECIMAL_FAN_SPEED%)"
|
||||
}
|
||||
|
||||
# Apply fan control to a specified value
|
||||
#
|
||||
# This function sets the fan speed to a user-specified value using ipmitool.
|
||||
# It first checks if the input value is in hexadecimal format, and converts it
|
||||
# if necessary. Then it sends raw commands to iDRAC to set the fan control.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (VALUE): The desired fan speed value. Can be in decimal or hexadecimal format.
|
||||
# If in decimal, it will be converted to hexadecimal.
|
||||
#
|
||||
# Returns:
|
||||
# None
|
||||
#
|
||||
# Note:
|
||||
# This function uses the global variable $IDRAC_LOGIN_STRING for iDRAC login.
|
||||
function apply_fan_control_to_specified_value() {
|
||||
local VALUE=$1
|
||||
|
||||
# Check if the input value is a hexadecimal number, if not, convert it to hexadecimal
|
||||
if [[ $VALUE != 0x* ]]; then
|
||||
VALUE=$(convert_decimal_value_to_hexadecimal "$VALUE")
|
||||
fi
|
||||
|
||||
# Use ipmitool to send the raw command to set fan control to user-specified value
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x01 0x00 > /dev/null
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x02 0xff "$VALUE" > /dev/null
|
||||
}
|
||||
|
||||
# Calculate the interpolated fan speed based on CPU temperature
|
||||
#
|
||||
# This function calculates the interpolated fan speed based on the current CPU temperature
|
||||
# and predefined thresholds. It uses linear interpolation to adjust the fan speed
|
||||
# within a specified range when the CPU temperature exceeds a certain threshold.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (HIGHEST_TEMPERATURE): The current highest CPU/GPU temperature (in Celsius)
|
||||
# $2 (TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION): The lower temperature threshold for fan speed interpolation (in Celsius)
|
||||
# $3 (TEMPERATURE_THRESHOLD): The upper temperature threshold for fan speed interpolation (in Celsius)
|
||||
# $4 (LOCAL_DECIMAL_FAN_SPEED): The base fan speed (as a decimal percentage, 0-100)
|
||||
# $5 (LOCAL_DECIMAL_HIGH_FAN_SPEED): The maximum fan speed (as a decimal percentage, 0-100)
|
||||
#
|
||||
# Returns:
|
||||
# The calculated interpolated fan speed as a decimal percentage (0-100)
|
||||
# If the temperature is below or equal to the lower threshold, returns the base fan speed
|
||||
# If the temperature is above or equal to the upper threshold, returns the maximum fan speed
|
||||
#
|
||||
# Usage:
|
||||
# calculate_interpolated_fan_speed <highest_cpu_temp> <lower_threshold> <upper_threshold> <base_fan_speed> <max_fan_speed>
|
||||
#
|
||||
# Example:
|
||||
# calculate_interpolated_fan_speed 70 60 80 30 100
|
||||
function calculate_interpolated_fan_speed() {
|
||||
local HIGHEST_CPU_TEMPERATURE=$1
|
||||
local CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION=$2
|
||||
local CPU_TEMPERATURE_THRESHOLD=$3
|
||||
local LOCAL_DECIMAL_FAN_SPEED=$4
|
||||
local LOCAL_DECIMAL_HIGH_FAN_SPEED=$5
|
||||
|
||||
# If temperature is below or equal to the lower threshold, return the base fan speed
|
||||
if [ "$HIGHEST_CPU_TEMPERATURE" -le "$CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION" ]; then
|
||||
echo "$LOCAL_DECIMAL_FAN_SPEED"
|
||||
return
|
||||
fi
|
||||
|
||||
# If temperature is above or equal to the upper threshold, return the max fan speed
|
||||
if [ "$HIGHEST_CPU_TEMPERATURE" -ge "$CPU_TEMPERATURE_THRESHOLD" ]; then
|
||||
echo "$LOCAL_DECIMAL_HIGH_FAN_SPEED"
|
||||
return
|
||||
fi
|
||||
|
||||
# F1 - lower fan speed
|
||||
# F2 - higher fan speed
|
||||
# T_CPU - highest temperature of both CPUs (if only one exists that will be CPU1 temp value)
|
||||
# T1 - lower temperature threshold
|
||||
# T2 - higher temperature threshold
|
||||
# Fan speed = F1 + ( ( F2 - F1 ) * ( T_CPU - T1 ) / ( T2 - T1 ) )
|
||||
|
||||
local TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE=$((CPU_TEMPERATURE_THRESHOLD - CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION))
|
||||
local FAN_VALUE_TO_ADD=0
|
||||
|
||||
if [ $TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE -gt $FAN_VALUE_TO_ADD ]; then
|
||||
local TEMPERATURE_ABOVE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION=$((HIGHEST_CPU_TEMPERATURE - CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION))
|
||||
local FAN_WINDOW=$((LOCAL_DECIMAL_HIGH_FAN_SPEED - LOCAL_DECIMAL_FAN_SPEED))
|
||||
FAN_VALUE_TO_ADD=$((FAN_WINDOW * TEMPERATURE_ABOVE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION / TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE))
|
||||
fi
|
||||
|
||||
local DECIMAL_CURRENT_FAN_SPEED=$((LOCAL_DECIMAL_FAN_SPEED + FAN_VALUE_TO_ADD))
|
||||
echo $DECIMAL_CURRENT_FAN_SPEED
|
||||
}
|
||||
|
||||
|
||||
# Convert first parameter given ($DECIMAL_NUMBER) to hexadecimal
|
||||
# Usage : convert_decimal_value_to_hexadecimal $DECIMAL_NUMBER
|
||||
# Returns : hexadecimal value of DECIMAL_NUMBER
|
||||
function convert_decimal_value_to_hexadecimal () {
|
||||
local DECIMAL_NUMBER=$1
|
||||
local HEXADECIMAL_NUMBER=$(printf '0x%02x' $DECIMAL_NUMBER)
|
||||
echo $HEXADECIMAL_NUMBER
|
||||
}
|
||||
|
||||
# Retrieve temperature sensors data using ipmitool
|
||||
# Usage : retrieve_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
function retrieve_cpu_temperatures() {
|
||||
if (( $# != 2 )); then
|
||||
printf "Illegal number of parameters.\nUsage: retrieve_temperatures \$IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT \$IS_CPU2_TEMPERATURE_SENSOR_PRESENT" >&2
|
||||
return 1
|
||||
fi
|
||||
local IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=$1
|
||||
local IS_CPU2_TEMPERATURE_SENSOR_PRESENT=$2
|
||||
|
||||
local DATA=$(ipmitool -I $IDRAC_LOGIN_STRING sdr type temperature | grep degrees)
|
||||
|
||||
# Parse CPU data
|
||||
local CPU_DATA=$(echo "$DATA" | grep "3\." | grep -Po '\d{2}')
|
||||
CPU1_TEMPERATURE=$(echo $CPU_DATA | awk "{print \$$CPU1_TEMPERATURE_INDEX;}")
|
||||
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
||||
CPU2_TEMPERATURE=$(echo $CPU_DATA | awk "{print \$$CPU2_TEMPERATURE_INDEX;}")
|
||||
else
|
||||
CPU2_TEMPERATURE="-"
|
||||
fi
|
||||
|
||||
# Parse inlet temperature data
|
||||
INLET_TEMPERATURE=$(echo "$DATA" | grep Inlet | grep -Po '\d{2}' | tail -1)
|
||||
|
||||
# If exhaust temperature sensor is present, parse its temperature data
|
||||
if $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT; then
|
||||
EXHAUST_TEMPERATURE=$(echo "$DATA" | grep Exhaust | grep -Po '\d{2}' | tail -1)
|
||||
else
|
||||
EXHAUST_TEMPERATURE="-"
|
||||
fi
|
||||
}
|
||||
|
||||
# Prepare traps in case of container exit
|
||||
function graceful_exit() {
|
||||
apply_Dell_fan_control_profile
|
||||
|
||||
echo "/!\ WARNING /!\ Container stopped, Dell default dynamic fan control profile applied for safety."
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Helps debugging when people are posting their output
|
||||
function get_Dell_server_model() {
|
||||
IPMI_FRU_content=$(ipmitool -I $IDRAC_LOGIN_STRING fru 2>/dev/null) # FRU stands for "Field Replaceable Unit"
|
||||
# TODO - Add check if connection was established. There are a chance user type wrong login and pass. In my case it returns "Error: Unable to establish IPMI v2 / RMCP+ session"
|
||||
|
||||
SERVER_MANUFACTURER=$(echo "$IPMI_FRU_content" | grep "Product Manufacturer" | awk -F ': ' '{print $2}')
|
||||
SERVER_MODEL=$(echo "$IPMI_FRU_content" | grep "Product Name" | awk -F ': ' '{print $2}')
|
||||
|
||||
# Check if SERVER_MANUFACTURER is empty, if yes, assign value based on "Board Mfg"
|
||||
if [ -z "$SERVER_MANUFACTURER" ]; then
|
||||
SERVER_MANUFACTURER=$(echo "$IPMI_FRU_content" | tr -s ' ' | grep "Board Mfg :" | awk -F ': ' '{print $2}')
|
||||
fi
|
||||
|
||||
# Check if SERVER_MODEL is empty, if yes, assign value based on "Board Product"
|
||||
if [ -z "$SERVER_MODEL" ]; then
|
||||
SERVER_MODEL=$(echo "$IPMI_FRU_content" | tr -s ' ' | grep "Board Product :" | awk -F ': ' '{print $2}')
|
||||
fi
|
||||
}
|
||||
Reference in New Issue
Block a user