脚本背景&＃xff1a;

我所在的公司为运营CDN业务的IDC公司&＃xff0c;客户域名的流量图经常会出现毛刺&＃xff0c;但是服务的域名非常多&＃xff0c;每天挨个流量图看耗时耗力。因此用python写了个可以自动检测异常rrd里异常数值并发送报警邮件的脚本。


由于我们的rrd文件是以服务域名命名的&＃xff0c;所以先在相应的API上获取服务域名&＃xff0c;然后根据域名扫描rrd文件。我设的是扫描半小时的数值&＃xff0c;每10分钟执行一次&＃xff0c;大概有2000来个rrd文件&＃xff0c;执行一次6、7秒左右。


代码如下&＃xff1a;


#!/usr/bin/env python
#coding:utf-8
from pyrrd.graph import DEF,CDEF,AREA
from pyrrd.graph import Graph
from pyrrd.graph import ColorAttributes
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.p_w_picpath import MIMEImage
from datetime import datetime
import calendar
import os
import time
import urllib2
import smtplib
import email
import sys
def graphrrd(files):now_utc &＃61;calendar.timegm(datetime.utcnow().utctimetuple())def1 &＃61; DEF(rrdfile&＃61;files, vname&＃61;&＃39;back&＃39;,dsName&＃61;&＃39;RX&＃39;)def2 &＃61; DEF(rrdfile&＃61;files, vname&＃61;&＃39;CDN&＃39;,dsName&＃61;&＃39;TX&＃39;)cdef1 &＃61; CDEF(vname&＃61;&＃39;back_flow&＃39;,rpn&＃61;&＃39;%s,0.026,*&＃39; % def1.vname)cdef2 &＃61; CDEF(vname&＃61;&＃39;CDN_flow&＃39;,rpn&＃61;&＃39;%s,0.026,*&＃39; % def2.vname)area1 &＃61; AREA(defObj&＃61;cdef1, color&＃61;&＃39;#002A97FF&＃39;, legend&＃61;&＃39;back_flow&＃39;)area2 &＃61; AREA(defObj&＃61;cdef2, color&＃61;&＃39;#00CF00FF&＃39;, legend&＃61;&＃39;CDN_flow&＃39;)ca &＃61; ColorAttributes()ca.back &＃61; &＃39;#333333&＃39;ca.canvas &＃61; &＃39;#333333&＃39;ca.shadea &＃61; &＃39;#000000&＃39;ca.shadeb &＃61; &＃39;#111111&＃39;ca.mgrid &＃61; &＃39;#CCCCCC&＃39;ca.axis &＃61; &＃39;#FFFFFF&＃39;ca.frame &＃61; &＃39;#AAAAAA&＃39;ca.font &＃61; &＃39;#FFFFFF&＃39;ca.arrow &＃61; &＃39;#FFFFFF&＃39;graphfile &＃61; p_w_picpath_dirtitle_url&＃61;files[23:-4]g &＃61; Graph(graphfile, start&＃61; now_utc-43200, end&＃61; now_utc,vertical_label&＃61;&＃39;flow&＃39;,title&＃61;title_url )g.data.extend([def1, def2, cdef1, cdef2, area2, area1])g.write()
def connect():server&＃61;smtplib.SMTP(smtpserver)server.ehlo()server.login(smtpuser,smtppass)return server
def sendmessage(server,to,subj,content):msg &＃61; MIMEMultipart(&＃39;related&＃39;)msg[&＃39;Subject&＃39;] &＃61; subjmsg[&＃39;From&＃39;] &＃61; smtpusermsg[&＃39;To&＃39;] &＃61; tomsg[&＃39;Date&＃39;] &＃61; email.Utils.formatdate() msgText &＃61; MIMEText(content,"html", "utf-8")msg.attach(msgText)fp &＃61; open(p_w_picpath_dir, &＃39;rb&＃39;)msgImage &＃61; MIMEImage(fp.read())fp.close()msgImage.add_header(&＃39;Content-ID&＃39;, &＃39;&＃39;)msg.attach(msgImage)try:server.sendmail(smtpuser, to, msg.as_string())except Exception ,ex:print Exception,exprint &＃39;Error - send failed&＃39;
def aver(rrd_file,n&＃61;6):global dict_datasum1&＃61;0sum2&＃61;0sum3&＃61;0data &＃61; os.popen(&＃39;rrdtool fetch %s AVERAGE -s -1d | tail -%d | grep -v nan| grep -v RX &＃39; % (rrd_file,n)).readlines()if len(data)<(n/2):log("[ERRORS: %s] has not enough record ! please check it!!\n" % rrd_file)return []for i in data:if len(i) > 25:dict_data[i[:10]]&＃61;i.strip()[12:].split()for i in dict_data.values():try:sum1 &＃61; sum1&＃43;float(i[0])sum2 &＃61; sum2&＃43;float(i[1])sum3 &＃61; sum3&＃43;float(i[2])except:log(&＃39;%s %s\n&＃39; % (rrd_file,i))if sum2/len(data) <3500000000:log(&＃39;WARNING: %s was less then 200M\n&＃39; % rrd_file)return []return [sum1/len(data),sum2/len(data),sum3/len(data)]
def check(average):wrong_t&＃61;[]for key in dict_data:if float(dict_data[key][1])/average > 1.6:wrong_t.append(key)return wrong_tdef update(rrd_file,t,aver1,aver2,aver3):global textglobal dict_dataerrors_time&＃61;os.popen(&＃39;date -d "1970-01-01 UTC %s seconds"&＃39; % t).readline().strip()content &＃61; &＃39;

%s 异常信息:
    域名:        %s
    时间:        %s
    流量值:     回源带宽: %.2fM , cdn带宽 : %dM
                
rrd 异常信息:
    路径:        %s
    UTC 时间:    %s
    异常值:     [%s], [%s], [%s]

&＃39; % (rrd_file[23:-4],rrd_file[23:-4],errors_time,float(dict_data[t][0])*8/300000000,int(float(dict_data[t][1])*8/300000000),rrd_file,t,dict_data[t][0],dict_data[t][1],dict_data[t][2])write_error(&＃39;[ %s ]: at[ %s(%s) ],the value was [%s] [%s] [%s] \n&＃39; %(rrd_file,errors_time,t,dict_data[t][0],dict_data[t][1],dict_data[t][2]))text &＃61; text &＃43; content
def log(log_write):f &＃61; open(&＃39;%s/rrd_alt1.log&＃39; % rrd_bak, &＃39;a&＃39;) f.write(log_write)f.close()
def write_error(log_write):f &＃61; open(&＃39;%s/rrd_error1.log&＃39; % rrd_bak, &＃39;a&＃39;)f.write(log_write)f.closedef run_script(rrd_file):global to_allglobal textaver_rrd&＃61;aver(rrd_file)if len(aver_rrd) &＃61;&＃61; 0:return wrong_time&＃61;check(aver_rrd[1])if len(wrong_time)&＃61;&＃61;0:log(&＃39;[%s] no errors !\n&＃39; % (rrd_file))returnfor t in wrong_time:update(rrd_file,t,aver_rrd[0],aver_rrd[1],aver_rrd[2]) graphrrd(rrd_file)if text:for to in to_all:server&＃61;connect()sendmessage(server,to,subj,text)log(&＃39;sendmail to %s\n&＃39; % to)
if __name__&＃61;&＃61;&＃39;__main__&＃39;:p_w_picpath_time&＃61;time.strftime("%d-%H-%M")rrd_dir&＃61;&＃39;/data/rrd/db/1/billing&＃39;rrd_bak&＃61;&＃39;/data/rrd/db/1/billing/bak&＃39;smtpserver&＃61;&＃39;xxx&＃39;p_w_picpath_dir&＃61;&＃39;%s/rrdgraph_%s.png&＃39; % (rrd_bak,p_w_picpath_time)smtpuser&＃61;&＃39;xxx&＃39;smtppass&＃61;&＃39;yyy&＃39;to_all&＃61;[&＃39;xxx&＃39;,&＃39;yyy&＃39;]subj&＃61;&＃39;check the flow of CDN!!!!&＃39;while True:url_list&＃61;[]local_time &＃61; time.strftime("%m-%d %H:%M:%S")url&＃61;urllib2.urlopen(&＃39;xxx&＃39;).readlines()for u in url:a &＃61; "%s/%s.rrd" % (rrd_dir,u.strip())url_list.append(a)log("-"*60&＃43;"\n")log("the script run time at %s \n" % local_time)while len(url_list):text&＃61;&＃39;&＃39;dict_data&＃61;{}rrd_file &＃61; url_list.pop()if os.path.exists(rrd_file):run_script(rrd_file)else:continue log("-"*60&＃43;"\n")break

邮件截图

173433350.jpg