#!/bin/sh
ip=10.116.100.11
port=8088
export HADOOP_HOME=/app/hadoop/bin
rmstate1=$($HADOOP_HOME/yarn rmadmin -getServiceState rm1)
rmstate2=$($HADOOP_HOME/yarn rmadmin -getServiceState rm2)
if [ $rmstate1 == *active* ]
then
ip=10.116.100.10
fi
echo $ip
cd /home/yarn/queuecheck
rm -f maxmemory.txt
rm -f usedmemory.txt
rm -f rate.txt
rm -f qrate.txt
curl http://$ip:$port/ws/v1/cluster/scheduler | jq . | sed 's/\"//g' > scheduler.json
sed -i '1,5d' scheduler.json
sed -n -e :a -e '1,33!{P;N;D;};N;ba' scheduler.json > nscheduler.json
cat nscheduler.json | grep -i "queueName" | awk -F ": " '{print $2}' | awk -F "," '{print $1}' > qname.txt
cat nscheduler.json | grep -i "memory" | awk -F ": " '{print $2}' | awk -F "," '{print $1}' > memory.txt
line=$(cat memory.txt | wc -l)
for((k=2;k<$line;k=k+6))
do
sed -n "$k p" memory.txt | awk '{print $1}' >> maxmemory.txt
done
for((j=3;j<$line;j=j+6))
do
sed -n "$j p" memory.txt | awk '{print $1}' >> usedmemory.txt
done
uarr=($(cat usedmemory.txt | awk '{print $1}'))
marr=($(cat maxmemory.txt | awk '{print $1}'))
for((m=0;m<${#uarr[@]};m++))
do
for((n=0;n<${#marr[@]};n++))
do
if [ $m -eq $n ]
then
if [[ ${uarr[$m]} == *e* ]]
then
u1=$(echo ${uarr[$m]} | grep -i "e+0" | awk -F "e" '{print $1}')
u2=$(echo ${uarr[$m]} | grep -i "e+0" | awk -F "e" '{print $2}' | awk -F "+0" '{print $2}')
if [ $u2 -eq 6 ]
then
u3=`echo "$u1*1000000/1024" | bc`
elif [ $u2 -eq 7 ]
then
u3=`echo "$u1*10000000/1024" | bc`
elif [ $u2 -eq 8 ]
then
u3=`echo "$u1*100000000/1024" | bc`
fi
else
u3=`echo "sclae=2; ${uarr[$m]}/1024" | bc`
fi
if [[ ${marr[$n]} == *e* ]]
then
m1=$(echo ${marr[$n]} | grep -i "e+0" | awk -F "e" '{print $1}')
m2=$(echo ${marr[$n]} | grep -i "e+0" | awk -F "e" '{print $2}' | awk -F "+0" '{print $2}')
if [ $m2 -eq 6 ]
then
m3=`echo "$m1*1000000/1024" | bc`
elif [ $m2 -eq 7 ]
then
m3=`echo "$m1*10000000/1024" | bc`
elif [ $m2 -eq 8 ]
then
m3=`echo "$m1*100000000/1024" | bc`
fi
else
m3=`echo "sclae=2; ${marr[$n]}/1024" | bc`
fi
rate=$(printf "%.2f" `echo "scale=2;$u3/$m3"|bc`)
nrate=`echo "$rate*100" | bc`
echo $nrate >> rate.txt
fi
done
done
qarr=($(cat qname.txt | awk '{print $1}'))
rarr=($(cat rate.txt | awk '{print $1}'))
for((a=0;a<${#qarr[@]};a++))
do
for((b=0;b<${#rarr[@]};b++))
do
if [ $a -eq $b ]
then
echo ${qarr[$a]}":"${rarr[$b]} >> qrate.txt
fi
done
done
url="http://xxx/monitor/report/sendReport"
linenum=$(cat qrate.txt | wc -l)
let linenum+=1
echo $linenum
#上报数据
function postdata(){
for (( k = 1; k< $linenum ; k++ ))
do
key=$(echo $(sed -n "$k p" qrate.txt | awk -F ":" '{print "bdp.ops.queue_used##qname="$1"##cluster=bdp##env=prod"}') | awk '{gsub(/[[:blank:]]*/,"",$0);print $0;}' )
value=$(sed -n "$k p" qrate.txt | awk -F ":" '{print $2}')
data="{\"key\":\"$key\",\"value\":\"$value\"}"
echo $data
cmd="curl -s -X POST -H 'Content-type':'application/json' -d '${data}' ${url}"
sh -c "$cmd"
done
}
postdata
expression="x>90"
time=5
count=1
alertMode="1,2,3"
members="bdp-ops"
call_users="bdp-ops"
call_content="BDP离线集群Yarn队列使用率告警"
add_url="http://xxx/monitor/rule/addReport"
hnum=$(cat qrate.txt | wc -l)
let hnum+=1
echo $hnum
function queuealert(){
for(( n = 1;n<$hnum;n++))
do
key=$(sed -n "$n p" qrate.txt | awk -F ":" '{print "bdp.ops.queue_used##qname="$1"##cluster=bdp##env=prod"}')
str1=$(sed -n "$n p" qrate.txt | awk -F ":" '{print $1}')
str2="的使用率大于90%,请检查Yarn UI作业运行情况"
description="队列"${str1}${str2}
add_data="{\"id\":\"$key\",\"expression\":\"$expression\",\"time\":$time,\"count\":$count,\"alertMode\":\"$alertMode\",\"members\":\"$members\",\"description\":\"$description\",\"call_users\":\"$call_users\",\"call_content\":\"$call_content\"}"
echo $add_data
add_cmd="curl -s -X POST -H 'Content-type':'application/json' -d '${add_data}' ${add_url}"
sh -c "$add_cmd"
done
}
queuealert
