LOG收集系统(一)：原日志至收集

Date: 20140207
Auth: Jin

设置一个LOG收集系统
1. 收集原生&＃xff08;不解析&＃xff0c;不压缩&＃xff09;的业务日志和WEB日志&＃xff08;NGINX,PHP&＃xff09;
2. 提供给开发&＃xff0c;测试直接阅读和下载

需求分析
原生日志&＃xff0c;所以不需要其他程序介入&＃xff0c;需要收集和阅读下载&＃xff0c;具体分析开发测试人员拿到日子自行处理
1、收集可以通过FTP采集
2、阅读和下载可以通过WEB形式
3、收集周期可以通过crontab控制脚本实现, 初步设置为每小时
4、核心是收集日子的脚本&＃xff0c;考虑到扩展和重复使用&＃xff0c;我使用python编写d

步骤
一、搭建FTP服务器

1.install

yum install pure-ftpd

2.config

mkdir /data/ftproot/logs

chown webroot.webroot /data/ftproot/logs

vim /etc/pure-ftpd/pure-ftpd.conf

注意流量参数

Bind 10.0.0.221,21
ChrootEveryone yes
BrokenClientsCompatibility yes
Daemonize yes
MaxClientsPerIP 20
VerboseLog yes
DisplayDotFiles no
AnonymousOnly no
NoAnonymous yes
SyslogFacility none
DontResolve yes
MaxIdleTime 15
LimitRecursion 10000 8
AnonymousCanCreateDirs no
MaxLoad 4
PassivePortRange 45000 50000
#AnonymousRatio 1 10
#UserRatio 1 10
AntiWarez yes
#AnonymousBandwidth 200
UserBandwidth 8
Umask 133:02
MinUID 100
AllowUserFXP no
AllowAnonymousFXP no
ProhibitDotFilesWrite no
ProhibitDotFilesRead no
AutoRename yes
AnonymousCantUpload yes
AltLog clf:/var/log/pureftpd.log
PureDB /etc/pure-ftpd/pureftpd.pdb
MaxDiskUsage 99
CreateHomeDir no
CustomerProof yes

View Code

 3.start ftp seriver

/etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]

chkconfig pure-ftpd on

4.add account

# pure-pw useradd logsftp -u 9999 -g 9999 -d /data/ftproot/logs
Password:
Enter it again:

# pure-pw mkdb
# /etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]

5.test

on opensuse desktop test

# zypper install lftp

lftp 10.0.0.221
lftp 10.0.0.221:~> user logsftp
Password:
lftp logsftp&＃64;10.0.0.221:~> ls
ls: Login failed: 530 &＃xfffd;&＃xfffd;֤ʧ&＃xfffd;ܣ&＃xfffd;&＃xfffd;&＃xfffd;Ǹ
lftp logsftp&＃64;10.0.0.221:~>

二、搭建WEB服务器
DOCMENTROOT为

注意点&＃xff1a;

autoindex_localtime on;
参考&＃xff1a;http://blog.chinaunix.net/uid-26719405-id-3508444.html
autoindex_localtime on;
默认为off&＃xff0c;显示的文件时间为GMT时间。
改为on后&＃xff0c;显示的文件时间为文件的服务器时间

三、代码编写

无LOG版本

加入计划任务

5 * * * * /home/gbin/logcollect.py >> /home/gbin/logcollect.log 2>&1

计划任务报错 登录用户问题

删除userName &＃61; os.getlogin()

• 配置文件logcollect.ini

• 函数和对象&＃xff1a;gbopt.py

  #!/usr/bin/env python
  # coding&＃61;utf8
  &＃39;&＃39;&＃39;
  Created on 2014-03-05 &＃64;author: Jin &＃39;&＃39;&＃39; __version__ &＃61; "0.0.1#date:2014-03-05" __all__ &＃61; [&＃39;readConfig&＃39;, &＃39;ftpPutGb&＃39;, &＃39;OptFile&＃39; ] import os
  import sys
  import ConfigParser
  import ftplib
  import time
  import socket
  from ftplib import FTP
  from time import strftime as printtime
  from time import sleep as waitprogName &＃61; &＃39;logcollect&＃39;
  userName &＃61; os.getlogin()
  hostName &＃61; socket.gethostname()
  pwdDir &＃61; os.getcwd()
  workDir &＃61; os.path.dirname(sys.argv[0])configFile &＃61; progName&＃43;&＃39;.ini&＃39;
  todayDate &＃61; time.strftime("%Y%m%d", time.localtime())if workDir &＃61;&＃61; &＃39;&＃39; or workDir &＃61;&＃61; &＃39;.&＃39;:progDir &＃61; pwdDir
  elif workDir.startswith(&＃39;/&＃39;):progDir &＃61; workDir
  elif workDir.startswith(&＃39;./&＃39;):progDir &＃61; pwdDir&＃43;workDir.lstrip(&＃39;.&＃39;)
  else:progDir &＃61; pwdDir&＃43;&＃39;/&＃39;&＃43;workDirconfPath &＃61; progDir&＃43;&＃39;/&＃39;&＃43;configFile
  logPath &＃61; progDir&＃43;&＃39;/&＃39;&＃43;progName&＃43;&＃39;.log&＃39;def readConfig(filename&＃61;&＃39;&＃39;,section&＃61;&＃39;&＃39;):&＃39;&＃39;&＃39;read ini config return dict&＃39;&＃39;&＃39;Call &＃61; &＃39;Call &＃39;&＃43;sys._getframe().f_code.co_nameif not os.path.isfile(filename):print "%s: Error, File %s is not exists,Please check it!" % (Call,filename)sys.exit(10)else:try:Config &＃61; ConfigParser.ConfigParser()Config.read(filename)sections &＃61; Config.sections()configDict &＃61; {}for conf in Config.items(section):ckey &＃61; conf[0]cvalue &＃61; conf[1]configDict[ckey] &＃61; cvalueexcept ConfigParser.ParsingError,e:print "%s: Read<%s> Section<%s> Parsing Error with reason<%s>!" % (Call,filename,section,&＃39;&＃39;.join( repr(e).split(&＃39;\n&＃39;)))sys.exit(11)except Exception, e:print "%s: Read<%s> Section<%s> Exception error with reason<%s>!" % (Call,filename,section,&＃39;&＃39;.join( repr(e).split(&＃39;\n&＃39;)))sys.exit(12)else:return configDictdef ftpPutGb(filename,inittime,server&＃61;None,username&＃61;None,password&＃61;None,localdir&＃61;None,remotedir&＃61;&＃39;testdir&＃39;,debuglevel&＃61;0,model&＃61;1,retrytime&＃61;10,timeout&＃61;120):"""Put file to FTP server,use local pathexitcode introduce: timeout | 99connect failed | 21user or passwd error | 22create dir fails | 23change dir fails | 24read file fails | 25transfer file fails | 26any ftp transfer fails | 27server or username or password not exist | 28call ftpPutGb(&＃39;/home/jin/code/python/tt.ini&＃39;,&＃39;127.0.0.1&＃39;,&＃39;logsftp&＃39;,&＃39;passwd&＃39;,remotedir&＃61;&＃39;testdir1/testdir2/testdir3&＃39;,model&＃61;0)"""Call &＃61; &＃39;Call &＃39;&＃43;sys._getframe().f_code.co_nameif not os.path.isfile(filename):print "Notice: %s is not exists!" % filenamereturn Falseif not localdir:localdir&＃61;os.path.dirname(filename)filename&＃61;os.path.basename(filename)uploadfile&＃61;&＃39;STOR &＃39;&＃43;filenameftp&＃61;FTP()bufsize&＃61;1024os.chdir(localdir)ftp.set_debuglevel(debuglevel)remotedirList&＃61;remotedir.split(&＃39;/&＃39;)appName&＃61;remotedirList[0]machineName&＃61;remotedirList[1]dayName&＃61;remotedirList[2]if server and username and password:print printtime(&＃39;%Y-%m-%d %H:%M:%S&＃39;),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) Begin." % (username,filename,server,remotedir)try:ftp.connect(server)except socket.error:waittime&＃61;inittimewait(retrytime)waittime&＃43;&＃61;retrytimeif waittime&＃61;&＃61;timeout:print "%s: Put %s to FTP(%s) timeout(%s),exit!" % (Call,filename,server,timeout)sys.exit(20)print "Notice: Retry Put %s to FTP(%s) again after %s s" % (filename,server,retrytime)ftpPutGb(filename,waittime,server,username,password,localdir,remotedir,debuglevel,model,retrytime,timeout)except ftplib.error_perm,e:print "Error: Connect to FTP(%s) failed with<%s>" % (server,e)sys.exit(21)else:try:ftp.login(username,password)ftp.set_pasv(model)except ftplib.error_perm,e:print "Error: User(%s) Login FTP(%s) failed with<%s>" % (username,server,e)sys.exit(22)else:try:try:ftp.mkd(appName)except ftplib.error_perm,e:passfinally:ftp.cwd(appName)try:ftp.mkd(machineName)except ftplib.error_perm,e:passfinally:ftp.cwd(machineName)try:ftp.mkd(dayName)except ftplib.error_perm,e:passfinally:ftp.cwd(dayName)try:file_handler &＃61; open(filename,&＃39;rb&＃39;)except ftplib.error_perm,e:print "Error: User(%s) on FTP(%s) Open file(%s) failed with %s!" % (username,server,filename,e)sys.exit(25)else:try:ftp.storbinary(uploadfile,file_handler,bufsize)except ftplib.error_perm,e:print "Error: User(%s) on FTP(%s) storbinary file(%s) failed with %s!" % (username,server,filename,e)sys.exit(26)except Exception, e:print "Error: User(%s) Put file(%s) to FTP(%s:%s) failed exit with %s!" % (username,filename,server,remotedir,e)sys.exit(27)else:print printtime(&＃39;%Y-%m-%d %H:%M:%S&＃39;),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) successful." % (username,filename,server,remotedir)return Truefinally:ftp.set_debuglevel(0)ftp.close()else:print "Error: server(%s) or username(%s) or password(%s) is not exist!" % (server,username,password)sys.exit(28)class OptFile(file):&＃39;&＃39;&＃39;log file object&＃39;&＃39;&＃39;def __init__(self,path,remotedir):self.filePath &＃61; pathself.remotedir &＃61; remotedirglobalDict &＃61; readConfig(confPath,&＃39;global&＃39;)self.ftpIp &＃61; globalDict[&＃39;ip&＃39;]self.ftpUser &＃61; globalDict[&＃39;username&＃39;]self.ftpPass &＃61; globalDict[&＃39;password&＃39;]self.ftpMode &＃61; globalDict[&＃39;mode&＃39;]file.__init__def put(self):ftpPutGb(self.filePath,0,self.ftpIp,self.ftpUser,self.ftpPass,remotedir&＃61;self.remotedir,model&＃61;self.ftpMode)def main():pass
  if __name__ &＃61;&＃61; &＃39;__main__&＃39;:main()

   

• 调用脚本&＃xff1a;logcollect.py