Add disk temperature monitoring, trend tracking and email alarms
Fan speed now takes the loudest request from CPU/GPU and from each drive interpolated against its own reported temperature limit, so bay position and drive technology stop mattering. Reported limits are clamped by class because WD Reds report the SCT critical limit (85) rather than an operating maximum. Adds a rolling temperature history and four alarms (absolute, rising trend, SMART health, disk count), rate limited per key with a one hour cooldown, delivered by mail through the host relay. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zeb Hering
215
functions.sh
Executable file
215
functions.sh
Executable file
@@ -0,0 +1,215 @@
|
||||
# Define global functions
|
||||
# This function applies Dell's default dynamic fan control profile
|
||||
function apply_Dell_fan_control_profile() {
|
||||
# Use ipmitool to send the raw command to set fan control to Dell default
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x01 0x01 > /dev/null
|
||||
CURRENT_FAN_CONTROL_PROFILE="Dell default dynamic fan control profile"
|
||||
}
|
||||
|
||||
#Get the TEMP of the TESLA P4 installed in this servers
|
||||
#Should probably set a variable so this function can check temps of any GPU using nvidia-smi
|
||||
retrieve_gpu_temperature() {
|
||||
if [ -x /usr/bin/nvidia-smi ]; then
|
||||
# Get the index of the Tesla P4 GPU using nvidia-smi
|
||||
index=$(nvidia-smi --query-gpu=name,index --format=csv | grep "Tesla P4" | cut -d ',' -f 2)
|
||||
else
|
||||
index=""
|
||||
fi
|
||||
|
||||
if [ "$index" != "" ]; then
|
||||
# Get the temperature of the Tesla P4 GPU using nvidia-smi and the index obtained above
|
||||
GPU_TEMPERATURE=$(nvidia-smi --query-gpu=temperature.gpu --format=csv | head -$(( $(($index + 2)) )) | tail -1)
|
||||
else
|
||||
GPU_TEMPERATURE="0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Apply user-defined fan control settings
|
||||
#
|
||||
# This function applies user-defined fan control settings based on the fan speed.
|
||||
# It handles both decimal and hexadecimal fan speed inputs, converting between them as needed.
|
||||
# The function then applies the fan control and updates the current fan control profile.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (LOCAL_FAN_SPEED): The desired fan speed. Can be in decimal (0-100) or hexadecimal (0x00-0x64) format.
|
||||
#
|
||||
# Global variables used:
|
||||
# CURRENT_FAN_CONTROL_PROFILE: Updated with the current fan control profile description.
|
||||
#
|
||||
# Returns:
|
||||
# None.
|
||||
function apply_user_fan_control() {
|
||||
local LOCAL_FAN_SPEED=$1
|
||||
|
||||
if [[ $LOCAL_FAN_SPEED == 0x* ]]; then
|
||||
local LOCAL_DECIMAL_FAN_SPEED
|
||||
LOCAL_DECIMAL_FAN_SPEED=$(printf '%d' "$LOCAL_FAN_SPEED")
|
||||
local LOCAL_HEXADECIMAL_FAN_SPEED=$LOCAL_FAN_SPEED
|
||||
else
|
||||
local LOCAL_DECIMAL_FAN_SPEED=$LOCAL_FAN_SPEED
|
||||
local LOCAL_HEXADECIMAL_FAN_SPEED
|
||||
LOCAL_HEXADECIMAL_FAN_SPEED=$(convert_decimal_value_to_hexadecimal "$LOCAL_FAN_SPEED")
|
||||
fi
|
||||
|
||||
apply_fan_control_to_specified_value "$LOCAL_HEXADECIMAL_FAN_SPEED"
|
||||
CURRENT_FAN_CONTROL_PROFILE="User static fan control profile ($LOCAL_DECIMAL_FAN_SPEED%)"
|
||||
}
|
||||
|
||||
# Apply fan control to a specified value
|
||||
#
|
||||
# This function sets the fan speed to a user-specified value using ipmitool.
|
||||
# It first checks if the input value is in hexadecimal format, and converts it
|
||||
# if necessary. Then it sends raw commands to iDRAC to set the fan control.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (VALUE): The desired fan speed value. Can be in decimal or hexadecimal format.
|
||||
# If in decimal, it will be converted to hexadecimal.
|
||||
#
|
||||
# Returns:
|
||||
# None
|
||||
#
|
||||
# Note:
|
||||
# This function uses the global variable $IDRAC_LOGIN_STRING for iDRAC login.
|
||||
function apply_fan_control_to_specified_value() {
|
||||
local VALUE=$1
|
||||
|
||||
# Check if the input value is a hexadecimal number, if not, convert it to hexadecimal
|
||||
if [[ $VALUE != 0x* ]]; then
|
||||
VALUE=$(convert_decimal_value_to_hexadecimal "$VALUE")
|
||||
fi
|
||||
|
||||
# Use ipmitool to send the raw command to set fan control to user-specified value
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x01 0x00 > /dev/null
|
||||
ipmitool -I $IDRAC_LOGIN_STRING raw 0x30 0x30 0x02 0xff "$VALUE" > /dev/null
|
||||
}
|
||||
|
||||
# Calculate the interpolated fan speed based on CPU temperature
|
||||
#
|
||||
# This function calculates the interpolated fan speed based on the current CPU temperature
|
||||
# and predefined thresholds. It uses linear interpolation to adjust the fan speed
|
||||
# within a specified range when the CPU temperature exceeds a certain threshold.
|
||||
#
|
||||
# Parameters:
|
||||
# $1 (HIGHEST_TEMPERATURE): The current highest CPU/GPU temperature (in Celsius)
|
||||
# $2 (TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION): The lower temperature threshold for fan speed interpolation (in Celsius)
|
||||
# $3 (TEMPERATURE_THRESHOLD): The upper temperature threshold for fan speed interpolation (in Celsius)
|
||||
# $4 (LOCAL_DECIMAL_FAN_SPEED): The base fan speed (as a decimal percentage, 0-100)
|
||||
# $5 (LOCAL_DECIMAL_HIGH_FAN_SPEED): The maximum fan speed (as a decimal percentage, 0-100)
|
||||
#
|
||||
# Returns:
|
||||
# The calculated interpolated fan speed as a decimal percentage (0-100)
|
||||
# If the temperature is below or equal to the lower threshold, returns the base fan speed
|
||||
# If the temperature is above or equal to the upper threshold, returns the maximum fan speed
|
||||
#
|
||||
# Usage:
|
||||
# calculate_interpolated_fan_speed <highest_cpu_temp> <lower_threshold> <upper_threshold> <base_fan_speed> <max_fan_speed>
|
||||
#
|
||||
# Example:
|
||||
# calculate_interpolated_fan_speed 70 60 80 30 100
|
||||
function calculate_interpolated_fan_speed() {
|
||||
local HIGHEST_CPU_TEMPERATURE=$1
|
||||
local CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION=$2
|
||||
local CPU_TEMPERATURE_THRESHOLD=$3
|
||||
local LOCAL_DECIMAL_FAN_SPEED=$4
|
||||
local LOCAL_DECIMAL_HIGH_FAN_SPEED=$5
|
||||
|
||||
# If temperature is below or equal to the lower threshold, return the base fan speed
|
||||
if [ "$HIGHEST_CPU_TEMPERATURE" -le "$CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION" ]; then
|
||||
echo "$LOCAL_DECIMAL_FAN_SPEED"
|
||||
return
|
||||
fi
|
||||
|
||||
# If temperature is above or equal to the upper threshold, return the max fan speed
|
||||
if [ "$HIGHEST_CPU_TEMPERATURE" -ge "$CPU_TEMPERATURE_THRESHOLD" ]; then
|
||||
echo "$LOCAL_DECIMAL_HIGH_FAN_SPEED"
|
||||
return
|
||||
fi
|
||||
|
||||
# F1 - lower fan speed
|
||||
# F2 - higher fan speed
|
||||
# T_CPU - highest temperature of both CPUs (if only one exists that will be CPU1 temp value)
|
||||
# T1 - lower temperature threshold
|
||||
# T2 - higher temperature threshold
|
||||
# Fan speed = F1 + ( ( F2 - F1 ) * ( T_CPU - T1 ) / ( T2 - T1 ) )
|
||||
|
||||
local TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE=$((CPU_TEMPERATURE_THRESHOLD - CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION))
|
||||
local FAN_VALUE_TO_ADD=0
|
||||
|
||||
if [ $TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE -gt $FAN_VALUE_TO_ADD ]; then
|
||||
local TEMPERATURE_ABOVE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION=$((HIGHEST_CPU_TEMPERATURE - CPU_TEMPERATURE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION))
|
||||
local FAN_WINDOW=$((LOCAL_DECIMAL_HIGH_FAN_SPEED - LOCAL_DECIMAL_FAN_SPEED))
|
||||
FAN_VALUE_TO_ADD=$((FAN_WINDOW * TEMPERATURE_ABOVE_THRESHOLD_FOR_FAN_SPEED_INTERPOLATION / TEMPERATURE_INTERPOLATION_ACTIVATION_RANGE))
|
||||
fi
|
||||
|
||||
local DECIMAL_CURRENT_FAN_SPEED=$((LOCAL_DECIMAL_FAN_SPEED + FAN_VALUE_TO_ADD))
|
||||
echo $DECIMAL_CURRENT_FAN_SPEED
|
||||
}
|
||||
|
||||
|
||||
# Convert first parameter given ($DECIMAL_NUMBER) to hexadecimal
|
||||
# Usage : convert_decimal_value_to_hexadecimal $DECIMAL_NUMBER
|
||||
# Returns : hexadecimal value of DECIMAL_NUMBER
|
||||
function convert_decimal_value_to_hexadecimal () {
|
||||
local DECIMAL_NUMBER=$1
|
||||
local HEXADECIMAL_NUMBER=$(printf '0x%02x' $DECIMAL_NUMBER)
|
||||
echo $HEXADECIMAL_NUMBER
|
||||
}
|
||||
|
||||
# Retrieve temperature sensors data using ipmitool
|
||||
# Usage : retrieve_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
|
||||
function retrieve_cpu_temperatures() {
|
||||
if (( $# != 2 )); then
|
||||
printf "Illegal number of parameters.\nUsage: retrieve_temperatures \$IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT \$IS_CPU2_TEMPERATURE_SENSOR_PRESENT" >&2
|
||||
return 1
|
||||
fi
|
||||
local IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=$1
|
||||
local IS_CPU2_TEMPERATURE_SENSOR_PRESENT=$2
|
||||
|
||||
local DATA=$(ipmitool -I $IDRAC_LOGIN_STRING sdr type temperature | grep degrees)
|
||||
|
||||
# Parse CPU data
|
||||
local CPU_DATA=$(echo "$DATA" | grep "3\." | grep -Po '\d{2}')
|
||||
CPU1_TEMPERATURE=$(echo $CPU_DATA | awk "{print \$$CPU1_TEMPERATURE_INDEX;}")
|
||||
if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
|
||||
CPU2_TEMPERATURE=$(echo $CPU_DATA | awk "{print \$$CPU2_TEMPERATURE_INDEX;}")
|
||||
else
|
||||
CPU2_TEMPERATURE="-"
|
||||
fi
|
||||
|
||||
# Parse inlet temperature data
|
||||
INLET_TEMPERATURE=$(echo "$DATA" | grep Inlet | grep -Po '\d{2}' | tail -1)
|
||||
|
||||
# If exhaust temperature sensor is present, parse its temperature data
|
||||
if $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT; then
|
||||
EXHAUST_TEMPERATURE=$(echo "$DATA" | grep Exhaust | grep -Po '\d{2}' | tail -1)
|
||||
else
|
||||
EXHAUST_TEMPERATURE="-"
|
||||
fi
|
||||
}
|
||||
|
||||
# Prepare traps in case of container exit
|
||||
function graceful_exit() {
|
||||
apply_Dell_fan_control_profile
|
||||
|
||||
echo "/!\ WARNING /!\ Container stopped, Dell default dynamic fan control profile applied for safety."
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Helps debugging when people are posting their output
|
||||
function get_Dell_server_model() {
|
||||
IPMI_FRU_content=$(ipmitool -I $IDRAC_LOGIN_STRING fru 2>/dev/null) # FRU stands for "Field Replaceable Unit"
|
||||
# TODO - Add check if connection was established. There are a chance user type wrong login and pass. In my case it returns "Error: Unable to establish IPMI v2 / RMCP+ session"
|
||||
|
||||
SERVER_MANUFACTURER=$(echo "$IPMI_FRU_content" | grep "Product Manufacturer" | awk -F ': ' '{print $2}')
|
||||
SERVER_MODEL=$(echo "$IPMI_FRU_content" | grep "Product Name" | awk -F ': ' '{print $2}')
|
||||
|
||||
# Check if SERVER_MANUFACTURER is empty, if yes, assign value based on "Board Mfg"
|
||||
if [ -z "$SERVER_MANUFACTURER" ]; then
|
||||
SERVER_MANUFACTURER=$(echo "$IPMI_FRU_content" | tr -s ' ' | grep "Board Mfg :" | awk -F ': ' '{print $2}')
|
||||
fi
|
||||
|
||||
# Check if SERVER_MODEL is empty, if yes, assign value based on "Board Product"
|
||||
if [ -z "$SERVER_MODEL" ]; then
|
||||
SERVER_MODEL=$(echo "$IPMI_FRU_content" | tr -s ' ' | grep "Board Product :" | awk -F ': ' '{print $2}')
|
||||
fi
|
||||
}
|
||||
Reference in New Issue
Block a user