#!/bin/bash
# +----------------------------------------------------------------------------
# | chkApache.sh
# | Written for www.radixhosting.com
# | 
# | - Performs a basic scan to check the Apache load
# | - Forcefully restarts Apache if necessary
# | - Sends a report to the system administrator if actions taken
# +----------------------------------------------------------------------------

###############################################################################
# *** START OF CONFIGURATION SETTINGS ***
###############################################################################

# The delay between integrity checks
DELAY=3s

# Minimum server load (5 min. average) for the script to run (integer value)
MIN_LOAD=4

# Maximum % CPU all httpd processes are allowed to use
MAX_CPU=90

# Maximum % memory all httpd processes are allowed to use
MAX_MEM=110

# Kill all httpd processes and restart Apache when a limit has been reached
RESTART_HTTPD=1

# Send a report to the system administrator when a limit has been reached
SEND_REPORT=1

# System administrator to send reports to
ADMIN_MAIL=root

###############################################################################
# *** END OF CONFIGURATION SETTINGS ***
###############################################################################

while [ 1 ]
do

if [ `cat /proc/loadavg | gawk -F "." '{ print $1 }'` -ge $MIN_LOAD ]
then
  KILL=0
  echo "The minimum server load has been reached, performing Apache integrity scan..."
  read REACHED_CPU CUR_CPU < <(ps aux | grep httpd | grep -v grep | gawk -vlimit=$MAX_CPU '{ total+=$3 } END { if (total>limit) { print 1, total } else { print 0, total } }')
  if [ $REACHED_CPU -eq 1 ]
  then
    echo "The CPU limit has been reached. Apache is using $CUR_CPU (limit: $MAX_CPU)."
    KILL=1
  else
    echo "The CPU limit has NOT been reached. Apache is using $CUR_CPU (limit: $MAX_CPU)."
  fi
  read REACHED_MEM CUR_MEM < <(ps aux | grep httpd | grep -v grep | gawk -vlimit=$MAX_MEM '{ total+=$4 } END { if (total>limit) { print 1, total } else { print 0, total } }')
  if [ $REACHED_MEM -eq 1 ]
  then
    echo "The memory limit has been reached. Apache is using $CUR_MEM (limit: $MAX_MEM)."
    KILL=1
  else
    echo "The memory limit has NOT been reached. Apache is using $CUR_MEM (limit: $MAX_MEM)."
  fi
  if [ $KILL -eq 1 ]
  then
    if [ $SEND_REPORT -eq 1 ]
    then
      echo "Sending report to the system administrator..."
      mail -s "chkApache.sh alert on `hostname`" $ADMIN_MAIL <<< cat <<EOF
******************************************************
chkApache.sh report
******************************************************

*** APACHE LOAD

CPU usage: $CUR_CPU (limit: $MAX_CPU)
Mem usage: $CUR_MEM (limit: $MAX_MEM)

*** CURRENT LOAD

`cat /proc/loadavg`

*** MEMORY STATISTICS

`free -m`

*** APACHE STATUS REPORT

`lynx -dump http://127.0.0.1/whm-server-status`

*** TOP PROCESSES LIST

`top -b -n 1`

*** NETSTAT HTTP CONNECTIONS

`netstat -a | grep :http`

*** APACHE PROCESSES

`ps aux | grep httpd`

******************************************************
EOF
    fi
    if [ $RESTART_HTTPD -eq 1 ]
    then
      echo "Trying to kill all httpd processes..."
      COUNTER=0
      while [ $COUNTER -lt 50 ]; do
        killall -15 httpd
        killall -9 httpd
        if [ $? -ne 0 ]
        then
          echo "All httpd processes were killed."
          COUNTER=100
        else
          COUNTER=`expr $COUNTER + 1`
        fi
      done
      if [ $COUNTER -ne 100 ]
      then
        echo "WARNING: Failed to kill all httpd processes!"
      fi
      echo "Restarting httpd..."
      service httpd startssl
      echo "Done."
      sleep 5s
    fi
  fi
else
  echo "The minimum server load has not been reached."
fi

sleep $DELAY

done