#!/bin/bash

source /root/fan_speed/functions.sh

# Trap the signals for container exit and run graceful_exit function
trap 'graceful_exit' SIGINT SIGQUIT SIGTERM

# Variable defintions
# LOW_FAN_SPEED = Lowest fan speed
# HIGH_FAN_SPEED = max fan speed. (0-100)
# IDRAC_HOST = local or IP of iDRAC
# IDRAC_USERNAME = username of iDRAC
# IDRAC_PASSWORD = password if iDRAC
# CHECK_INTERVAL = time to wait in seconds before performing another temp check (if doing GPU might want to keep this low like 5 seconds)
# CPU_TEMPERATURE_THRESHOLD = max CPU temp before disabling this automation and setting idrac back to auto
# GPU_TEMPERATURE_THRESHOLD = max GPU TEMP before running a kill on the container running LLM
# TABLE_HEADER_PRINT_INTERVAL = Number of loops before we output temps


IDRAC_HOST=local
IDRAC_USERNAME=root
IDRAC_PASSWORD=calvin
CHECK_INTERVAL=10 #takes about 8 seconds to query all data
LOW_TEMPERATURE_THRESHOLD=45 #only decimal numbers
CPU_TEMPERATURE_THRESHOLD=90 #only decimal numbers
GPU_TEMPERATURE_THRESHOLD=75 #only decimal numbers
LOW_FAN_SPEED=18 #only decimal numbers
HIGH_FAN_SPEED=50 #only decimal numbers
TABLE_HEADER_PRINT_INTERVAL=2


#######################################
# Check if the iDRAC host is set to 'local' or not then set the IDRAC_LOGIN_STRING accordingly
if [[ $IDRAC_HOST == "local" ]]; then
  # Check that the Docker host IPMI device (the iDRAC) has been exposed to the Docker container
  if [ ! -e "/dev/ipmi0" ] && [ ! -e "/dev/ipmi/0" ] && [ ! -e "/dev/ipmidev/0" ]; then
    echo "/!\ Could not open device at /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0, check that you added the device to your Docker container or stop using local mode. Exiting." >&2
    exit 1
  fi
  IDRAC_LOGIN_STRING='open'
else
  echo "iDRAC/IPMI username: $IDRAC_USERNAME"
  echo "iDRAC/IPMI password: $IDRAC_PASSWORD"
  IDRAC_LOGIN_STRING="lanplus -H $IDRAC_HOST -U $IDRAC_USERNAME -P $IDRAC_PASSWORD"
fi
#######################################

#######################################
# This script ONLY runs on Dell Servers
get_Dell_server_model
if [[ ! $SERVER_MANUFACTURER == "DELL" ]]; then
  echo "/!\ Your server isn't a Dell product. Exiting." >&2
  exit 1
fi
#######################################

#######################################
#****Probably don't need this any more****
# Prepare, format and define initial variables
# Check if LOW_FAN_SPEED variable is in hexadecimal format. If not, convert it to hexadecimal
if [[ $LOW_FAN_SPEED == 0x* ]]; then
  readonly DECIMAL_LOW_FAN_SPEED=$(printf '%d' $LOW_FAN_SPEED)
  readonly HEXADECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
else
  readonly DECIMAL_LOW_FAN_SPEED=$LOW_FAN_SPEED
  readonly HEXADECIMAL_LOW_FAN_SPEED=$(convert_decimal_value_to_hexadecimal $LOW_FAN_SPEED)
fi
#######################################

#######################################
# This returns the lowest temp between CPU and GPU highest thresholds
# Used to calcuate fan speed interpolation
# I picked the lowest temp as the fan speeds will ramp up faster to account for the lower temp
if [ $CPU_TEMPERATURE_THRESHOLD -le $GPU_TEMPERATURE_THRESHOLD ]; then
  HIGH_TEMPERATURE_THRESHOLD=$CPU_TEMPERATURE_THRESHOLD
else
  HIGH_TEMPERATURE_THRESHOLD=$GPU_TEMPERATURE_THRESHOLD
fi
#######################################

#######################################
# Determine where to get CPU temps from iDRAC
# If server model is Gen 14 (*40) or newer
if [[ $SERVER_MODEL =~ .*[RT][[:space:]]?[0-9][4-9]0.* ]]; then
  DELL_POWEREDGE_GEN_14_OR_NEWER=true
  CPU1_TEMPERATURE_INDEX=2
  CPU2_TEMPERATURE_INDEX=4
else
  DELL_POWEREDGE_GEN_14_OR_NEWER=false
  CPU1_TEMPERATURE_INDEX=1
  CPU2_TEMPERATURE_INDEX=2
fi
#######################################

#######################################
# Check if sensors are present on the server
IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=true
IS_CPU2_TEMPERATURE_SENSOR_PRESENT=true
retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
if [ -z "$EXHAUST_TEMPERATURE" ]; then
  echo "No exhaust temperature sensor detected."
  IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT=false
fi
if [ -z "$CPU2_TEMPERATURE" ]; then
  echo "No CPU2 temperature sensor detected."
  IS_CPU2_TEMPERATURE_SENSOR_PRESENT=false
fi
#######################################

#######################################
# Print out config information before starting
#echo "Server Manufacturer: $SERVER_MANUFACTURER"
#echo "Server Model: $SERVER_MODEL"
#echo "iDRAC Host: $IDRAC_HOST"
#echo "Check Interval: $CHECK_INTERVAL"
#echo "Low Temp Threshold: $LOW_TEMPERATURE_THRESHOLD"
#echo "CPU High Temp threshold: $CPU_TEMPERATURE_THRESHOLD"
#echo "GPU High Temp Threshold: $GPU_TEMPERATURE_THRESHOLD"
#echo "Low Fan Speed: $LOW_FAN_SPEED"
#echo "High Fan Speed: $HIGH_FAN_SPEED"
#######################################

#######################################
# Loop (this is where the magic happens)
while true; do
  # Sleep for the specified interval before taking another reading
  sleep $CHECK_INTERVAL &
  SLEEP_PROCESS_PID=$!

  #Get Temps
  retrieve_cpu_temperatures $IS_EXHAUST_TEMPERATURE_SENSOR_PRESENT $IS_CPU2_TEMPERATURE_SENSOR_PRESENT
  retrieve_gpu_temperature

  i=0
  # Get highest CPU Temp and check if any of them are over heating
  HIGHEST_CPU_TEMPERATURE=$CPU1_TEMPERATURE
  if $IS_CPU2_TEMPERATURE_SENSOR_PRESENT; then
    if [ $CPU2_TEMPERATURE -gt $CPU1_TEMPERATURE ]; then
      HIGHEST_CPU_TEMPERATURE=$CPU2_TEMPERATURE
    fi
  fi

  #echo $HIGHEST_CPU_TEMPERATURE


  # Define Function to check for if CPU temps are greater than the threshold
  function CPU_OVERHEAT() { [ $HIGHEST_CPU_TEMPERATURE -gt $CPU_TEMPERATURE_THRESHOLD ]; }

  # Define function to check if GPU temperature are above the threshold
  function GPU_OVERHEAT() { [ $GPU_TEMPERATURE -gt $GPU_TEMPERATURE_THRESHOLD ]; }

  # Check if CPU is overheating then apply Dell default dynamic fan control profile if true
  # or set CPU fan based on interpolated temps
  if CPU_OVERHEAT; then
    apply_Dell_fan_control_profile
    if ! $IS_DELL_FAN_CONTROL_PROFILE_APPLIED; then
      IS_DELL_FAN_CONTROL_PROFILE_APPLIED=true
      COMMENT="CPU temperature is too high, Dell default dynamic fan control profile applied for safety"
    fi
  else
    #this combines CPU and GPU temps and outputs which one is highest
    HIGHEST_TEMPERATURE=$HIGHEST_CPU_TEMPERATURE
    if [ $GPU_TEMPERATURE -gt $HIGHEST_CPU_TEMPERATURE ]; then
      HIGHEST_TEMPERATURE=$GPU_TEMPERATURE
    fi
    if [ "$HIGHEST_TEMPERATURE" -gt "$LOW_TEMPERATURE_THRESHOLD" ]; then
	  DECIMAL_CURRENT_FAN_SPEED=$(calculate_interpolated_fan_speed $HIGHEST_TEMPERATURE $LOW_TEMPERATURE_THRESHOLD $HIGH_TEMPERATURE_THRESHOLD $LOW_FAN_SPEED $HIGH_FAN_SPEED)
    else
      DECIMAL_CURRENT_FAN_SPEED=$DECIMAL_LOW_FAN_SPEED
    fi
	apply_user_fan_control "$DECIMAL_CURRENT_FAN_SPEED"
	COMMENT=$(echo "CPU1:$CPU1_TEMPERATURE | CPU2:$CPU2_TEMPERATURE | GPU:$GPU_TEMPERATURE | Inlet:$INLET_TEMPERATURE | Exhaust:$EXHAUST_TEMPERATURE | Fan Speed:$DECIMAL_CURRENT_FAN_SPEED")
  fi

  if GPU_OVERHEAT; then
    COMMENT="GPU temperature is too high, killing CT(s) to prevent GPU damage"
	# This will eventually have a function to kill the CT(s) that use the Tesla P4
  fi

  # Print temperatures, active fan control profile and comment if any change happened during last time interval
  if [ $i -eq $TABLE_HEADER_PRINT_INTERVAL ]; then
    #echo "                     ------- Temperatures -------"
    #$echo "    Date & time      Inlet  CPU 1  CPU 2  Exhaust          Active fan speed profile          Third-party PCIe card Dell default cooling response  Comment"
    i=0
  fi
  #printf "%19s  %3d°C  %3d°C  %3s°C  %5s°C  %40s  %51s  %s\n" "$(date +"%d-%m-%Y %T")" $INLET_TEMPERATURE $CPU1_TEMPERATURE "$CPU2_TEMPERATURE" "$EXHAUST_TEMPERATURE" "$CURRENT_FAN_CONTROL_PROFILE" "$THIRD_PARTY_PCIE_CARD_DELL_DEFAULT_COOLING_RESPONSE_STATUS" "$COMMENT"
  z_date=$(date +"%Y-%m-%d %H:%M:%S")
  echo "$z_date || $COMMENT" >> /root/fan_speed/log/fan_speed.log
  ((i++))
  wait $SLEEP_PROCESS_PID
done