#!/bin/sh ip=10.116.100.11 port=8088 export HADOOP_HOME=/app/hadoop/bin rmstate1=$($HADOOP_HOME/yarn rmadmin -getServiceState rm1) rmstate2=$($HADOOP_HOME/yarn rmadmin -getServiceState rm2) if [ $rmstate1 == *active* ] then ip=10.116.100.10 fi echo $ip cd /home/yarn/queuecheck rm -f maxmemory.txt rm -f usedmemory.txt rm -f rate.txt rm -f qrate.txt curl http://$ip:$port/ws/v1/cluster/scheduler | jq . | sed 's/\"//g' > scheduler.json sed -i '1,5d' scheduler.json sed -n -e :a -e '1,33!{P;N;D;};N;ba' scheduler.json > nscheduler.json cat nscheduler.json | grep -i "queueName" | awk -F ": " '{print $2}' | awk -F "," '{print $1}' > qname.txt cat nscheduler.json | grep -i "memory" | awk -F ": " '{print $2}' | awk -F "," '{print $1}' > memory.txt line=$(cat memory.txt | wc -l) for((k=2;k<$line;k=k+6)) do sed -n "$k p" memory.txt | awk '{print $1}' >> maxmemory.txt done for((j=3;j<$line;j=j+6)) do sed -n "$j p" memory.txt | awk '{print $1}' >> usedmemory.txt done uarr=($(cat usedmemory.txt | awk '{print $1}')) marr=($(cat maxmemory.txt | awk '{print $1}')) for((m=0;m<${#uarr[@]};m++)) do for((n=0;n<${#marr[@]};n++)) do if [ $m -eq $n ] then if [[ ${uarr[$m]} == *e* ]] then u1=$(echo ${uarr[$m]} | grep -i "e+0" | awk -F "e" '{print $1}') u2=$(echo ${uarr[$m]} | grep -i "e+0" | awk -F "e" '{print $2}' | awk -F "+0" '{print $2}') if [ $u2 -eq 6 ] then u3=`echo "$u1*1000000/1024" | bc` elif [ $u2 -eq 7 ] then u3=`echo "$u1*10000000/1024" | bc` elif [ $u2 -eq 8 ] then u3=`echo "$u1*100000000/1024" | bc` fi else u3=`echo "sclae=2; ${uarr[$m]}/1024" | bc` fi if [[ ${marr[$n]} == *e* ]] then m1=$(echo ${marr[$n]} | grep -i "e+0" | awk -F "e" '{print $1}') m2=$(echo ${marr[$n]} | grep -i "e+0" | awk -F "e" '{print $2}' | awk -F "+0" '{print $2}') if [ $m2 -eq 6 ] then m3=`echo "$m1*1000000/1024" | bc` elif [ $m2 -eq 7 ] then m3=`echo "$m1*10000000/1024" | bc` elif [ $m2 -eq 8 ] then m3=`echo "$m1*100000000/1024" | bc` fi else m3=`echo "sclae=2; ${marr[$n]}/1024" | bc` fi rate=$(printf "%.2f" `echo "scale=2;$u3/$m3"|bc`) nrate=`echo "$rate*100" | bc` echo $nrate >> rate.txt fi done done qarr=($(cat qname.txt | awk '{print $1}')) rarr=($(cat rate.txt | awk '{print $1}')) for((a=0;a<${#qarr[@]};a++)) do for((b=0;b<${#rarr[@]};b++)) do if [ $a -eq $b ] then echo ${qarr[$a]}":"${rarr[$b]} >> qrate.txt fi done done url="http://xxx/monitor/report/sendReport" linenum=$(cat qrate.txt | wc -l) let linenum+=1 echo $linenum #上报数据 function postdata(){ for (( k = 1; k< $linenum ; k++ )) do key=$(echo $(sed -n "$k p" qrate.txt | awk -F ":" '{print "bdp.ops.queue_used##qname="$1"##cluster=bdp##env=prod"}') | awk '{gsub(/[[:blank:]]*/,"",$0);print $0;}' ) value=$(sed -n "$k p" qrate.txt | awk -F ":" '{print $2}') data="{\"key\":\"$key\",\"value\":\"$value\"}" echo $data cmd="curl -s -X POST -H 'Content-type':'application/json' -d '${data}' ${url}" sh -c "$cmd" done } postdata expression="x>90" time=5 count=1 alertMode="1,2,3" members="bdp-ops" call_users="bdp-ops" call_content="BDP离线集群Yarn队列使用率告警" add_url="http://xxx/monitor/rule/addReport" hnum=$(cat qrate.txt | wc -l) let hnum+=1 echo $hnum function queuealert(){ for(( n = 1;n<$hnum;n++)) do key=$(sed -n "$n p" qrate.txt | awk -F ":" '{print "bdp.ops.queue_used##qname="$1"##cluster=bdp##env=prod"}') str1=$(sed -n "$n p" qrate.txt | awk -F ":" '{print $1}') str2="的使用率大于90%,请检查Yarn UI作业运行情况" description="队列"${str1}${str2} add_data="{\"id\":\"$key\",\"expression\":\"$expression\",\"time\":$time,\"count\":$count,\"alertMode\":\"$alertMode\",\"members\":\"$members\",\"description\":\"$description\",\"call_users\":\"$call_users\",\"call_content\":\"$call_content\"}" echo $add_data add_cmd="curl -s -X POST -H 'Content-type':'application/json' -d '${add_data}' ${add_url}" sh -c "$add_cmd" done } queuealert