#!/bin/sh
#-------------------------------------------------------------------------------
# charon_logevent
#-------------------------------------------------------------------------------
# Script is running within crontab that periodically check for log events and
# send events in bulk mode
#-------------------------------------------------------------------------------
# Copyright (C) 2013-2022 STROMASYS.
# All rights reserved.
#
VERSION=1.11

for P in `ls /etc/profile.d/charon*`;do . $P;done
. `dirname $0`/charon_common

MONBUGCHECK="/root/.charon.logmon.no.bugcheck"
MONCPUHALTED="/root/.charon.logmon.no.cpuhalted"

function PrepareMail
{
  cat ${FILE}.tmp | sed -e "s=^=<li>=g" -e "s=$=</li><br>=g" \
      -e "s=:INFO :=<font color\=#${COL_INFO}>:INFO :</font>=g" \
      -e "s=:WARN :=<font color\=#${COL_WARN}>:WARN :</font>=g" \
      -e "s=:ERROR:=<font color\=#${COL_ERROR}>:ERROR:</font>=g" \
      -e "s=:FATAL:=<font color\=#${COL_FATAL}><b>:FATAL:</b></font>=g" \
      -e "s=\(^.*\)\(CPU.* BUGCHECK\)\(.*$\)=\1<font color\=#FF8040>\2</font>\3=g" \
      -e "s=\(^.*\)\(CPU.* Halted\)\(.*$\)=\1<font color\=#FF8040>\2</font>\3=g" \
      >>${MAILFILE}
  echo "</ul>" >>${MAILFILE}
  rm -f ${FILE}.tmp

  ALERTLOG=${CHARONDIR}/log/charon_alerts.log
  ( echo "--------------------------------------------------------------------------------"; \
    echo "`date +%Y%m%d:%H%M%S`:ALERT:$1"; \
    cat ${MAILFILE} | sed -e "s=<br>==g" \
      -e "s=<b>==g"     -e "s=</b>==g" \
      -e "s=<li>==g"    -e "s=</li>==g" \
      -e "s=<ul>==g"    -e "s=</ul>==g" \
      -e "s=</font>==g" -e "s=<font .*>==g" \
      -e "s=</a>==g"    -e "s=<a .*>==g" ) >>${ALERTLOG}
}

function SendMail
{
  KITV=`cat ${CHARONDIR}/utils/kit.version 2>/dev/null | cut -f1 -d';'`
  KITD=`cat ${CHARONDIR}/utils/kit.version 2>/dev/null | cut -f2 -d';'`

  TMPMAIL=${MAILFILE}.tmp
  MAILFROM=`cat ${MAILFRFILE} 2>/dev/null`
  MAILFROM=`cat ${MAILFRFILE} 2>/dev/null`
  if test -z "${MAILFROM}"
  then
    echo "From: `whoami`@`hostname`" >${TMPMAIL}
  else
    if test -n "`echo ${MAILFROM} | grep '@'`"
    then
      echo "From: ${MAILFROM}" >${TMPMAIL}
    else
      echo "From: ${MAILFROM}<`whoami`@`hostname`>" >${TMPMAIL}
    fi
  fi

  echo "To: ${MAILTO}" >>${TMPMAIL}
  echo "Subject: [CHARON] $1" >>${TMPMAIL}

  case "$2"
  in
    "LOW")
      echo "Importance: low" >>${TMPMAIL}
      echo "X-Priority: 5" >>${TMPMAIL}
      ;;
    "HIGH")
      echo "Importance: high" >>${TMPMAIL}
      echo "X-Priority: 1" >>${TMPMAIL}
      ;;
  esac

  if test "${MAILMODE}" = "TEXT"
  then
    cat ${MAILFILE} | sed -e "s=<br>==g" \
        -e "s=<b>==g"     -e "s=</b>==g" \
        -e "s=<li>==g"    -e "s=</li>==g" \
        -e "s=<ul>==g"    -e "s=</ul>==g" \
        -e "s=</font>==g" -e "s=<font .*>==g" \
        -e "s=</a>==g"    -e "s=<a .*>==g"    >>${TMPMAIL}
    if test -s ${CHARONDIR}/utils/charon_check.mailfooter
    then
      echo >>${TMPMAIL}
      echo "------------------------------------------------" >>${TMPMAIL}
      cat ${CHARONDIR}/utils/charon_check.mailfooter >>${TMPMAIL}
    fi

    echo >>${TMPMAIL}
    echo "(Kit version ${KITV:-unknown}, dated ${KITD:-unknown})" >>${TMPMAIL}

    /usr/sbin/sendmail -t -i <${TMPMAIL} 
    RET=$?
    test ${RET} = 0 && rm -f ${TMPMAIL}

  else
    echo "Mime-Version: 1.0;" >>${TMPMAIL}
    echo "Content-Type: text/html; charset=\"ISO-8859-1\";" >>${TMPMAIL}
    echo "Content-Disposition: inline;" >>${TMPMAIL}
    echo "<html>" >>${TMPMAIL}
    cat ${MAILFILE} >>${TMPMAIL}
    if test -s ${CHARONDIR}/utils/charon_check.mailfooter
    then
      echo "<br>" >>${TMPMAIL}
      echo "<hr color="#3aaadc">" >>${TMPMAIL}
      cat ${CHARONDIR}/utils/charon_check.mailfooter | sed "s=$=<br>=g" >>${TMPMAIL}
    fi
    echo "</html>" >>${TMPMAIL}

    echo "<br>" >>${TMPMAIL}
    echo "(<font color=#3E3E40>Kit version ${KITV:-unknown}, dated ${KITD:-unknown}</font>)<br>" >>${TMPMAIL}

    /usr/sbin/sendmail -t -i <${TMPMAIL} 
    RET=$?
    test ${RET} = 0 && rm -f ${TMPMAIL}

  fi
  rm -f ${MAILFILE}
}

test -d ${CHARONDIR}/utils/events || mkdir ${CHARONDIR}/utils/events 

if test -e ${CHARONDIR}/utils/charon_check.mailto
then
  MAILTO=`grep -v ^# ${CHARONDIR}/utils/charon_check.mailto | sort -u | tr "\n" ","|sed "s=,$==g"`
else
  MAILTO="root@`hostname`,support@stromasys.com"
fi

MAILMODE=`cat /root/.charonmailmode 2>/dev/null`
test "${MAILMODE}" != "TEXT" && MAILMODE="HTML"

MAILFRFILE="${CHARONDIR}/utils/charon_check.mailfrom"

MAILFILE="/tmp/`basename $0`.`date +"%Y%m%d.%H%M%S.%N"`"

CHKEVENT=/root/.charonlogevent
test -e ${CHKEVENT} || echo 2 >${CHKEVENT}
typeset -i LVLEVENT=`cat ${CHKEVENT}`

typeset -i NBR_INFO=0
typeset -i NBR_WARN=0
typeset -i NBR_ERROR=0
typeset -i NBR_FATAL=0

FILCOUNT="/tmp/`basename $0`.counter"
if test `ls -1 ${CHARONDIR}/utils/events/*.partlog 2>/dev/null | wc -l` -eq 0
then
  rm -f ${FILCOUNT}.* 2>/dev/null
else
  for FILE in `ls -1 ${CHARONDIR}/utils/events/*.partlog 2>/dev/null`
  do
    HOST=`basename ${FILE}|cut -f1 -d '.'`
    mv -f ${FILE} ${FILE}.tmp
    FILCOUNT="/tmp/`basename $0`.counter.${HOST}"

    NBR_INFO=`grep -c ':INFO :' ${FILE}.tmp`
    COL_INFO="000000"
    test ${NBR_INFO} -gt 0 && COL_INFO="2020FF"
    NBD_INFO=${NBR_INFO}
    if test ${LVLEVENT} -gt 1
    then
      NBD_INFO="Not monitored"
      test -e ${MONBUGCHECK}  || NBD_INFO="${NBD_INFO}. BUGCHECK messages monitored"
      test -e ${MONCPUHALTED} || NBD_INFO="${NBD_INFO}. CPUx Halted messages monitored"
    fi

    NBR_WARN=`grep -c ':WARN :' ${FILE}.tmp`
    COL_WARN="000000"
    test ${NBR_WARN} -gt 0 && COL_WARN="FF8040"
    NBD_WARN=${NBR_WARN}
    test ${LVLEVENT} -gt 2 && NBD_WARN="Not monitored"

    NBR_ERROR=`grep -c ':ERROR:' ${FILE}.tmp`
    COL_ERROR="000000"
    test ${NBR_ERROR} -gt 0 && COL_ERROR="FF2020"
    NBD_ERROR=${NBR_ERROR}
    test ${LVLEVENT} -eq 0 && NBD_ERROR="Not monitored"

    NBR_FATAL=`grep -c ':FATAL:' ${FILE}.tmp`
    COL_FATAL="000000"
    test ${NBR_FATAL} -gt 0 && COL_FATAL="C71585"
    NBD_FATAL=${NBR_FATAL}
    # test ${LVLEVENT} -eq 0 && NBD_FATAL="Not monitored"

    echo "`date +%s`, I:${NBD_INFO}, W:${NBD_WARN}, E:${NBD_ERROR}, F:${NBD_FATAL}" >>${FILCOUNT}

    #-----------------------------------------------------------------------
    # if FILCOUNT has more than 5 entries and its 1st entry is less than 20
    # minutes old then display alert and block next emails
    #-----------------------------------------------------------------------
    typeset -i CHKDATE=`date +%s`-`head -1 ${FILCOUNT} | cut -f1 -d','`
    NBLFILC=`cat ${FILCOUNT} | wc -l`

    PRIO="DEFAULT"
    test ${NBR_INFO}  -gt 0 && PRIO="DEFAULT"
    test ${NBR_WARN}  -gt 0 && PRIO="DEFAULT"
    test ${NBR_ERROR} -gt 0 && PRIO="HIGH"
    test ${NBR_FATAL} -gt 0 && PRIO="HIGH"

    case ${NBLFILC}
    in
      1|2|3|4)
        cat <<EOF >${MAILFILE}
<b><font color="#0080C0">Events count:</font></b><br>
INFORMATIONAL: <b><font color="#${COL_INFO}">${NBD_INFO}</font></b><br>
WARNING: <b><font color="#${COL_WARN}">${NBD_WARN}</font></b><br>
ERROR: <b><font color="#${COL_ERROR}">${NBD_ERROR}</font></b><br>
FATAL: <b><font color="#${COL_FATAL}">${NBD_FATAL}</font></b><br>
<br>
<b><font color="#0080C0">Events:</font></b><br>
<ul>
EOF
        PrepareMail
        SendMail "Events found in ${HOST}.log" "${PRIO}"
        ;;
      5)
        if test ${CHKDATE} -lt 1200
        then
          cat <<EOF >${MAILFILE}
<b><font color="#0080C0">Events count:</font></b><br>
INFORMATIONAL: <b><font color="#${COL_INFO}">${NBD_INFO}</font></b><br>
WARNING: <b><font color="#${COL_WARN}">${NBD_WARN}</font></b><br>
ERROR: <b><font color="#${COL_ERROR}">${NBD_ERROR}</font></b><br>
FATAL: <b><font color="#${COL_FATAL}">${NBD_FATAL}</font></b><br>
<br>
<b><font color="#E95F40">TOO MANY ALERTS, EMAILS PAUSED</font></b><br>
<b><font color="#E95F40">MANUAL CHECK OF THE LOG FILE REQUIRED</font></b><br>
<b><font color="#3E3E40">Emails will be released after 20 minutes or if no event found on next check</font></b><br>
<br>
<b><font color="#0080C0">Events:</font></b><br>
<ul>
EOF
          PrepareMail
          SendMail "Events found in ${HOST}.log (manual check required)" "HIGH"
        else
          rm -f ${FILCOUNT} 2>/dev/null
          cat <<EOF >${MAILFILE}
<b><font color="#0080C0">Events count:</font></b><br>
INFORMATIONAL: <b><font color="#${COL_INFO}">${NBD_INFO}</font></b><br>
WARNING: <b><font color="#${COL_WARN}">${NBD_WARN}</font></b><br>
ERROR: <b><font color="#${COL_ERROR}">${NBD_ERROR}</font></b><br>
FATAL: <b><font color="#${COL_FATAL}">${NBD_FATAL}</font></b><br>
<br>
<b><font color="#0080C0">Events:</font></b><br>
<ul>
EOF
          PrepareMail
          SendMail "Events found in ${HOST}.log (restored alerts)" "${PRIO}"
        fi
        ;;
      *)
        if test ${CHKDATE} -ge 1200
        then
          rm -f ${FILCOUNT} 2>/dev/null
          cat <<EOF >${MAILFILE}
<b><font color="#0080C0">Events count:</font></b><br>
INFORMATIONAL: <b><font color="#${COL_INFO}">${NBD_INFO}</font></b><br>
WARNING: <b><font color="#${COL_WARN}">${NBD_WARN}</font></b><br>
ERROR: <b><font color="#${COL_ERROR}">${NBD_ERROR}</font></b><br>
FATAL: <b><font color="#${COL_FATAL}">${NBD_FATAL}</font></b><br>
<br>
<b><font color="#0080C0">Events:</font></b><br>
<ul>
EOF
          PrepareMail
          SendMail "Events found in ${HOST}.log (restored alerts)" "${PRIO}"
        fi
        ;;
    esac
  done
fi

exit
