Daemon check sctipt

Linux

Daemon check sctipt

techapi 2025. 3. 12. 16:02

1. check_proc.sh

#!/bin/sh

while (true)

# ACTION=`bash /usr/local/daemon_chk/action.sh`

# DATE=`date +%Y%m%d`

# DATE_T=`date +%Y%m%d[%T]`

/bin/sh /usr/local/daemon_chk/action_10s.sh

sleep 10;

done

2. (2)초마다 실행

### check.sh 스크립트 ###

#!/bin/sh

for i in {1..30}; do

node /home/tobecold/blockchain/ice_http.js > /dev/null;

sleep 2;

done

### 실행 ###

nohup ./check.sh > /dev/null

3. action.sh

#!/bin/sh

# 20121024 Modified by BDH

export LANG=C

TODAY=`date +%y/%m/%d" "%H:%M:%S`

TODAY_LOG=`date +%Y%m%d`

MIN_CHK=`date +%M`

HOURMIN_CHK=`date +%H%M`

ServerIP=`ifconfig eth0 | grep "\<inet addr\>" | awk '{ gsub("addr:", "" ) ; print $2}'`

chk_dir="/usr/local/daemon_chk"

ftp_check_port="21"

ftp_check_proc="proftpd"

ftp_daemon="/etc/rc.d/init.d/proftpd"

mail_check_port="25"

mail_check_proc="sendmail"

mail_daemon="/etc/rc.d/init.d/sendmail"

MAILADDRESS="system@ www.test.com .com "

oldlog_remove(){

cd ${chk_dir}

/usr/bin/find ${chk_dir}/log -type f -mtime 7 | /usr/bin/xargs /bin/rm -f

}

check_log_dir() {

if [ ! -d /usr/local/daemon_chk/log ];then

mkdir /usr/local/daemon_chk/log

chmod 700 /usr/local/daemon_chk/log

if [ ! -f /usr/local/manage_hosting/exclude_domain ];then

touch /usr/local/manage_hosting/exclude_domain

chmod 700 /usr/local/manage_hosting/exclude_domain

if [ ! -f /usr/local/daemon_chk/exclude_domain ];then

mkdir /usr/local/daemon_chk/exclude

chmod 700 /usr/local/daemon_chk/exclude

}

IP_CONNTRACK_CHECK() {

IP_CONNTRACK_COUNT=`cat /proc/sys/net/ipv4/ip_conntrack_max`

if [ ${IP_CONNTRACK_COUNT} != "229376" ];then

echo "229376" > /proc/sys/net/ipv4/ip_conntrack_max

/sbin/sysctl -p

printf "$TODAY ip_conntrack_max value is Modified....\n" >> ${chk_dir}/log/${TODAY_LOG}

else

printf "$TODAY ip_conntrack_max value is OK....\n" >> ${chk_dir}/log/${TODAY_LOG}

}

CHECK_BAK_CHK() {

CURL_CHK=`ps aux|grep curl|grep -v grep| awk -F" " '{print $2}'`

if [ "" != "${CURL_CHK}" ]

then

pkill curl

else

echo "curl process none"

}

FTP_CHECK() {

BBNET=`${chk_dir}/bbnet localhost:$ftp_check_port`

RET=`echo $?`

FTPCHK=`netstat -ltn | awk '{print $4}' | grep ":\<$ftp_check_port\>"`

if [ "$RET" == "0" ] ; then

printf "$TODAY $ftp_check_proc is LIVING....\n" >> ${chk_dir}/log/${TODAY_LOG}

else

pstree > /tmp/${ftp_check_proc}_$$

if [ "$FTPCHK" ] ; then

mail -s "[ $ServerIP $ftp_check_proc BUSY !!! ] $TODAY" $MAILADDRESS < /tmp/${ftp_check_proc}_$$

printf "\nhurry! hurry! $TODAY $ftp_check_proc is BUSY!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

else

$ftp_daemon restart >> ${chk_dir}/log/${TODAY_LOG} 2>> /tmp/${ftp_check_proc}_$$

if [ "$?" -eq "0" ] ; then

mail -s "[ $ServerIP $ftp_check_proc RESTART !!! ] $TODAY" $MAILADDRESS < /tmp/${ftp_check_proc}_$$

printf "\nhurry! hurry! $TODAY $ftp_check_proc is RESTART!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

else

mail -s "[ $ServerIP $ftp_check_proc DOWN !!! ] $TODAY" $MAILADDRESS < /tmp/${ftp_check_proc}_$$

printf "\nhurry! hurry! $TODAY $ftp_check_proc is DOWN!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

rm -f /tmp/${ftp_check_proc}_$$

}

SENDMAIL_CHECK(){

MailCHK=`netstat -ltn | awk '{print $4}' | grep ":\<$mail_check_port\>"`

if ! [ "$MailCHK" ] ; then

pstree > /tmp/${mail_check_proc}_$$

printf "\n$TODAY sendmail is KILLED!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

$mail_daemon restart >> ${chk_dir}/log/${TODAY_LOG} 2>> /tmp/${mail_check_proc}_$$

sleep 1 ;

mail -s "[ $ServerIP $mail_check_proc RESTART !!! ] $TODAY " $MAILADDRESS < /tmp/${mail_check_proc}_$$

else

printf "$TODAY sendmail is LIVING....\n" >> ${chk_dir}/log/${TODAY_LOG}

rm -f /tmp/${mail_check_proc}_$$

}

PERL_CHECK() {

PERL_CHK=`ps -u nobody |grep perl| awk -F" " '{print $1}'`

if [ "" != "${PERL_CHK}" ]

then

ls -al /proc/${PERL_CHK} | mail -s "${ServerIP} perl이 돌고 있습니다" ${MAILADDRESS}

kill -9 ${PERL_CHK}

else

echo "이상 없습니다."

}

HACK_CHECK() {

for LIST in `ls -al /proc/ | grep "nobody" | awk -F" " '{print $8}'`

ls -al /proc/$LIST >> /tmp/web_hack.tmp

done

CHK_HACK=`cat /tmp/web_hack.tmp | grep deleted`

if [ "$CHK_HACK" == "" ] ; then

printf "$TODAY hack OK....\n" >> ${chk_dir}/log/${TODAY_LOG}

else

mail -s "[ $ServerIP Check web hack ] $TODAY" $MAILADDRESS < /tmp/web_hack.tmp

printf "$TODAY \nhurry! hurry! Check hack!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

rm -f /tmp/web_hack.tmp

}

UDP_CHECK() {

UDP_CHK=`netstat -napu |grep httpd|wc -l`

if [ "${UDP_CHK}" -gt 10 ]

then

printf "$TODAY udp check exdeed !! \n" >> ${chk_dir}/log/${TODAY_LOG}

netstat -napu |grep httpd > /tmp/udpchk.tmp

for PID in `netstat -napu|grep httpd|awk '{print $7}'|awk -F'/' '{print $1}'`

ls -al /proc/$PID | grep cwd >> /tmp/udpchk.tmp

done

mail -s "[ $ServerIP Check httpd UDP proc !! ] $TODAY" $MAILADDRESS < /tmp/udpchk.tmp

else

printf "$TODAY udp check OK....\n" >> ${chk_dir}/log/${TODAY_LOG}

rm -f /tmp/udpchk.tmp

}

USR_HTTPD_CHECK() {

if [ "${HTTPD_USER_CHK}" -gt 1 ]

then

printf "$TODAY httpd user account exist !! \n" >> ${chk_dir}/log/${TODAY_LOG}

echo "##### User Account Daemon #####" >> /tmp/usr_httpd_chk.tmp

ps aux |grep httpd |grep -Ev "grep|root|nobody" >> /tmp/usr_httpd_chk.tmp

echo "" >> /tmp/usr_httpd_chk.tmp

echo "##### User Account Daemon Running cwd" >> /tmp/usr_httpd_chk.tmp

for PID in `ps aux |grep httpd |grep -Ev "grep|root|nobody"|awk '{print $2}'`

ls -al /proc/$PID | grep cwd >> /tmp/usr_httpd_chk.tmp

kill -9 $PID

done

mail -s "[ $ServerIP httpd User Account Running Check !! ] $TODAY" $MAILADDRESS < /tmp/usr_httpd_chk.tmp

else

printf "$TODAY httpd user account OK....\n" >> ${chk_dir}/log/${TODAY_LOG}

rm -f /tmp/usr_httpd_chk.tmp

}

PROC_10s_CHECK() {

CHK_10s_PROC="check_proc.sh"

chk_proc_cnt=`ps aux |grep check_proc |grep -Ev "grep"|wc -l`

# httpd_chk_count=`tail -n 8 ${chk_dir}/log/${TODAY_LOG}|grep httpd|wc -l`

# if [ "${httpd_chk_count}" = "0" ]

# then

# ps aux|grep check_proc|grep -v grep |awk '{print $2}'|xargs kill -9

# printf "$TODAY 10s_check proc is Anomaly status!! \n" >> ${chk_dir}/log/${TODAY_LOG}

# fi

if [ "${chk_proc_cnt}" = "1" ]

then

printf "$TODAY 10s_check proc is OK!! \n" >> ${chk_dir}/log/${TODAY_LOG}

elif [ "${chk_proc_cnt}" = "0" ]

then

printf "$TODAY 10s_check proc is not exist!! \n" >> ${chk_dir}/log/${TODAY_LOG}

sh ${chk_dir}/${CHK_10s_PROC} &

ps aux |grep check_proc|grep -v grep > /tmp/proc_check_proc.tmp

mail -s "[ $ServerIP Process CHK proc Start !! ] $TODAY" $MAILADDRESS < /tmp/proc_check_proc.tmp

elif [ "${chk_proc_cnt}" -ge "2" ]

then

printf "$TODAY 10s_check proc is 2 or more exist!! \n" >> ${chk_dir}/log/${TODAY_LOG}

ps aux |grep check_proc|grep -v grep|awk '{print $2}'|xargs kill -9

sh ${chk_dir}/${CHK_10s_PROC} &

ps aux |grep check_proc|grep -v grep > /tmp/proc_check_proc.tmp

mail -s "[ $ServerIP Process CHK proc Start !! ] $TODAY" $MAILADDRESS < /tmp/proc_check_proc.tmp

rm -f /tmp/proc_check_proc.tmp

}

if [ "${HOURMIN_CHK}" -gt "0600" -a "${HOURMIN_CHK}" -lt "0610" ]

then

IP_CONNTRACK_CHECK

CHECK_BAK_CHK

if [ "${MIN_CHK}" = 00 ]

then

oldlog_remove

check_log_dir

if [ "${MIN_CHK}" = 00 -o "${MIN_CHK}" = 10 -o "${MIN_CHK}" = 20 -o "${MIN_CHK}" = 30 -o "${MIN_CHK}" = 40 -o "${MIN_CHK}" = 50 ]

then

FTP_CHECK

SENDMAIL_CHECK

#USR_HTTPD_CHECK

#PERL_CHECK

HACK_CHECK

#UDP_CHECK

PROC_10s_CHECK

3.1. action_10s.sh

#!/bin/sh

# 20121120 Modified by BDH

export LANG=C

TODAY=`date +%y/%m/%d" "%H:%M:%S`

TODAY_LOG=`date +%Y%m%d`

ServerIP=`ifconfig eth0 | grep "\<inet addr\>" | awk '{ gsub("addr:", "" ) ; print $2}'`

chk_dir="/usr/local/daemon_chk"

httpd_check_port="80"

httpd_check_proc="httpd"

httpd_daemon="/etc/rc.d/init.d/httpd"

LOADAVG="/proc/loadavg"

LOAD=`cat $LOADAVG | awk -F"." '{print $1}'`

KILLALL="/usr/bin/killall"

CROND="/etc/init.d/crond"

HOSTNAME=`/bin/hostname`

LYNX="/usr/bin/lynx"

MAILADDRESS="system@test.com "

SEC_CHK=`date +%S`

HTTPD_CHECK() {

BBNET=`$chk_dir/bbnet localhost:$httpd_check_port`

RET=`echo $?`

HTTPDCHK=`netstat -ltn | awk '{print $4}' | grep ":\<$httpd_check_port\>"`

HTTPDCHK2=`ps -ef | grep httpd | wc -l`

#CHK3=`pstree | grep httpd | grep -v sshd | awk -F "-" '{print $5}' | sed 's/*\[httpd\]//g'`

if [ "$RET" == "0" ] ; then

## httpd status ok

if [ $HTTPDCHK2 -gt "5" -a $HTTPDCHK2 -lt "400" ] ; then

printf "$TODAY $httpd_check_proc is LIVING....\n" >> $chk_dir/log/${TODAY_LOG}

else

pstree > /tmp/${httpd_check_proc}_$$

# MaxClient 400 over, httpd restart

if [ $HTTPDCHK2 -ge "400" ] ; then

$KILLALL -9 httpd

sleep 3

ipcs -s |sed "/em/d" | for i in `awk '{print $2}'` ; do ipcrm sem $i ; done

sleep 1

$httpd_daemon restart >> $chk_dir/log/${TODAY_LOG} 2>> /tmp/${httpd_check_proc}_$$

printf "\nhurry! hurry! $TODAY $httpd_check_proc is MaxClient $HTTPDCHK2 RESTART!!!!\n" >> $chk_dir/log/${TODAY_LOG}

mail -s "[ $ServerIP $httpd_check_proc MaxClient $HTTPDCHK2 RESTART !!! ] $TODAY" $MAILADDRESS < /tmp/${httpd_check_proc}_$$

# httpd ppid anomaly check, httpd restart

if [ $HTTPDCHK2 -le "5" ] ; then

$KILLALL -9 httpd

ipcs -s |sed "/em/d" | for i in `awk '{print $2}'` ; do ipcrm sem $i ; done

sleep 1

$httpd_daemon restart >> $chk_dir/log/${TODAY_LOG} 2>> /tmp/${httpd_check_proc}_$$

printf "\nhurry! hurry! $TODAY $httpd_check_proc is RESTART!!!!\n" >> $chk_dir/log/${TODAY_LOG}

mail -s "[ $ServerIP $httpd_check_proc RESTART !!! ] $TODAY" $MAILADDRESS < /tmp/${httpd_check_proc}_$$

## httpd status stopped

else

pstree > /tmp/${httpd_check_proc}_$$

# load 5 less httpd restart

if [ $LOAD -le "5" ] ; then

$httpd_daemon restart >> $chk_dir/log/${TODAY_LOG} 2>> /tmp/${httpd_check_proc}_$$

printf "\nhurry! hurry! $TODAY $httpd_check_proc isn't Checked. ReStart!!!!\n" >> $chk_dir/log/${TODAY_LOG}

mail -s "[ $ServerIP $httpd_check_proc isn't Checked, ReStart !!! ] $TODAY" $MAILADDRESS < /tmp/${httpd_check_proc}_$$

else

# load 5 less httpd can't restart

printf "\nhurry! hurry! $TODAY $httpd_check_proc isn't Checked. But Load Avg 5 over - ${LOAD}, httpd Can't Start !!!!\n" >> $chk_dir/log/${TODAY_LOG}

# mail -s "[ $ServerIP $httpd_check_proc isn't Checked, Load Avg ${LOAD} CHK plz !!! ] $TODAY" $MAILADDRESS < /tmp/${httpd_check_proc}_$$

rm -f /tmp/${httpd_check_proc}_$$

}

HTTPD_MEM_CHECK() {

ps aux|grep nobody|grep httpd|sort -k5 -r|awk '{print $2" "$6}' > /usr/local/daemon_chk/mem_chk.txt

exec < /usr/local/daemon_chk/mem_chk.txt

while read PID MEM

if [ ${MEM} -gt 100000 ]

then

# echo "${PID} ${MEM}"

/bin/kill -9 $PID

printf "$TODAY httpd RES 100M exdeed !! \n" >> ${chk_dir}/log/${TODAY_LOG}

/bin/rm -f /usr/local/daemon_chk/mem_chk.txt

done

}

HTTPD_LOAD_OVER() {

if [ -e $LOADAVG ]

then

if [ $LOAD -gt 20 ]

then

$KILLALL -9 httpd

printf "$TODAY Load Avg 20 over - ${LOAD} !! \n" >> ${chk_dir}/log/${TODAY_LOG}

Uptime=`/usr/bin/uptime > /tmp/httpd_load_over.tmp`

mail -s "$HOSTNAME Load Average $LOAD Over!!! HTTPD Down!!!" $MAILADDRESS < /tmp/httpd_load_over.tmp

rm -f /tmp/httpd_load_over.tmp

if [ "${SEC_CHK}" -ge "00" -a "${SEC_CHK}" -le "09" ]

then

$LYNX --dump "http://www.test.com/sms/linuxsmssend.php?Server_ip=$HOSTNAME&Process=$LOAD"

else

printf "$TODAY Load Avg status OK !! \n" >> ${chk_dir}/log/${TODAY_LOG}

else

echo LOADAVG File Missing

}

PHPINI_intergrity_check() {

#PHPINI=`/usr/local/php/bin/php -i |grep php.ini|awk '{print $6}'`

PHPINI="/usr/local/apache/conf/php.ini"

PHP_DIFF_LOG=php_ini_check

stat ${PHPINI}|grep -v "Access: [0-9]" > ${chk_dir}/${PHP_DIFF_LOG}

md5sum ${PHPINI} >> ${chk_dir}/${PHP_DIFF_LOG}

diff ${chk_dir}/${PHP_DIFF_LOG}_old ${chk_dir}/${PHP_DIFF_LOG}

RETN=`echo $?`

if [ "$RETN" == "0" ] ; then

printf "$TODAY php.ini File is OK....\n" >> ${chk_dir}/log/${TODAY_LOG}

else

printf "$TODAY php.ini File is anomaly!!!!!\n" >> ${chk_dir}/log/${TODAY_LOG}

mail -s "$HOSTNAME php.ini 파일변조 발생!! 서버확인 요망 !!!" system@test.com < ${PHPINI}

sleep 1

$LYNX --dump "http://www.test.com /sms/linux_php_modi_smssend.php?Server_ip=$HOSTNAME"

cat ${chk_dir}/${PHP_DIFF_LOG} > ${chk_dir}/${PHP_DIFF_LOG}_old

}

ROOT_PERM_CHECK() {

ROOT_PERM=`stat -c '%a' /`

if [ ${ROOT_PERM} != "751" ] ; then

printf "$TODAY / Perm ${ROOT_PERM} !!!\n" >> ${chk_dir}/log/${TODAY_LOG}

chmod 751 /

mail -s "$HOSTNAME / Perm Changed !!!" system@test.com < ${ROOT_PERM}

else

printf "$TODAY / Perm is OK - ${ROOT_PERM} \n" >> ${chk_dir}/log/${TODAY_LOG}

}

HTTPD_CHECK

HTTPD_MEM_CHECK

HTTPD_LOAD_OVER

#PHPINI_intergrity_check

#ROOT_PERM_CHECK

'Linux' 카테고리의 다른 글

root 파티션 용량 늘리기 (0)	2025.03.12
aws disk 용량 증설 (0)	2025.03.12
Aapache_logrotate script (0)	2025.03.12
PDFtk(PDF Toolkit) 설치 (0)	2025.03.12
ClamAV설치 (linux 백신) (0)	2025.03.12

현재글Daemon check sctipt

techapi 님의 블로그

개인적인 운영 자료이며, 기억을 정리하기 위해서 남겨 놓습니다.

MariaDB, 영상녹화, a.i코딩, Apache, script, tomcat, oracle, cursor, Linux, session test, passwd, iptables, aws, mod security, Python, redis, 백업, 노트북 분실, php, Trae,

Today :
Yesterday :

일	월	화	수	목	금	토
					1	2
3	4	5	6	7	8	9
10	11	12	13	14	15	16
17	18	19	20	21	22	23
24	25	26	27	28	29	30
31

techapi 님의 블로그