django+tornado实现实时查看远程日志的方法

import os
import sys
import tornado.websocket
import tornado.web
import tornado.ioloop
import redis
import salt.client

from tornado import gen
from tornado.escape import to_unicode

from logs.utility import get_last_lines
from logs import settings


class SubWebSocket(tornado.websocket.WebSocketHandler):
 """
 此handler处理远程日志查看
 """
 def open(self, *args, **kwargs):
  print("opened")

 @gen.coroutine
 def on_message(self, message):
  # 主机名，要查看的日志路径，运行脚本的命令这些信息从浏览器传过来
  hostname, log_path, cmd = message.split("||")
  local = salt.client.LocalClient()
  r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT,
        password=settings.REDIS_PASSWD, db=5)
  # 订阅频道，服务器和日志路径确定一个频道
  key = settings.LOG_KEY.format(server=hostname.strip(), log_path=log_path.strip())
  channel = r.pubsub()
  channel.subscribe(key)
  # 异步方式执行命令，远程运行脚本
  local.cmd_async(hostname, "cmd.run", [cmd])
  try:
   while True:
    data = channel.get_message()
    if not data:
     # 如果读取不到消息，间隔一定时间，避免无谓的CPU消耗
     yield gen.sleep(0.05)
     continue
    if data["type"] == "message":
     line = format_line(data["data"])
     self.write_message(line)
  except tornado.websocket.WebSocketClosedError:
   self.close()

 def on_close(self):
  global FLAG
  FLAG = False
  print("closed")


def format_line(line):
 line = to_unicode(line)
 if "INFO" in line:
  color = "#46A3FF"
 elif "WARN" in line:
  color = "#FFFF37"
 elif "ERROR" in line:
  color = "red"
 elif "CRITICAL" in line:
  color = "red"
 else:
  color = "#FFFFFF"

 return "{}".format(color, line)


class EchoWebSocket(tornado.websocket.WebSocketHandler):
 def open(self):
  print("WebSocket opened")

 @gen.coroutine
 def on_message(self, message):
  log = message
  print "log file: ", log

  try:
   with open(log, 'r') as f:
    for line in get_last_lines(f):
     line1 = format_line(line)
     self.write_message(line1)
    while True:
     line = f.readline()
     if not line:
      yield gen.sleep(0.05)
      continue
     self.write_message(format_line(line.strip()))
  except tornado.websocket.WebSocketClosedError as e:
   print e
   self.close()

 # def check_origin(self, origin):
 #  print origin, self.request.headers.get("Host")
 #  # super(EchoWebSocket, self).check_origin()
 #  return True

 def on_close(self):
  print("WebSocket closed")


class Application(tornado.web.Application):
 def __init__(self):
  handlers = [
   (r'/log/', MainHandler), # 提供浏览页面，页面中的JS与服务器建立连接
   (r'/log/local', EchoWebSocket), # 处理本地日志实时查看，比较简单
   (r'/log/remote', SubWebSocket), # 处理远程日志实时查看，稍微复杂
  ]
  settings = {
   "debug": True,
   "template_path": os.path.join(os.path.dirname(__file__), "templates"),
   "static_path": os.path.join(os.path.dirname(__file__), "static"),
  }
  super(Application, self).__init__(handlers, **settings)


class MainHandler(tornado.web.RequestHandler):
 def get(self):
  # 要查看的日志路径
  log = self.get_argument("log", None)
  # hostname实际上是saltstack中这台机器对应的minion id
  hostname = self.get_argument("hostname", None)
  # 本地日志还是远程日志
  type = self.get_argument("type", "local")
  # 运行读取实时日志的脚本，参数比较多，后面会有
  cmd = self.get_argument("cmd", "")
  cOntext= {
   "log": log,
   "hostname": hostname,
   "type": type,
   "cmd": cmd,
  }
  self.render("index.html", **context)

# encoding: utf-8

LOG_KEY = "logs:{server}:{log_path}"

LOG_NAME = "catalina.out"
TAIL_LINE_NUM = 20

REDIS_HOST = "127.0.0.1"
REDIS_PORT = "6379"
REDIS_PASSWD = None
REDIS_EXPIRE = 300

try:
 from local_settings import *
except ImportError:
 pass

 
 
 

 
 
 
 
 

  Stop
  Start
 

class LogView(KylinView):
 # 实时读取日志的脚本，事先使用saltstack批量传到各台服务器上
 client_path = "/tmp/logtail.py"

 def get(self, request):
  minion_id = request.GET.get("minion_id")
  cOntext= {
   "minion_id": minion_id,
   "tail_log_url": settings.TAIL_LOG_URL,
  }
  return render(request, "cmdb/log_view.html", context)

 def post(self, request):
  minion_id = request.POST.get("minion_id")
  log_path = request.POST.get("log_path")
  if not log_path:
   return JsonResponse({"success": False, "message": "请填写日志路径"})
  try:
   # 制定一开始读取的行数
   line_count = request.POST.get("line_count")
  except (TypeError, ValueError):
   return JsonResponse({"success": False, "message": "请输入正确的行数"})
  local = salt.client.LocalClient()
  # 确保saltstack能连通并且日志文件存在
  ret = local.cmd(minion_id, "file.file_exists", [log_path])
  if minion_id not in ret:
   return JsonResponse({"success": False, "message": "服务器无法连通"})
  if not ret[minion_id]:
   return JsonResponse({"success": False, "message": "日志文件不存在"})
  # 组成命令的各个参数，redis信息需要和tornado配置文件中的redis信息一致
  cmd = "{} {} {} {} {} {} {} {}".format(
   settings.PYTHON_BIN, self.client_path, minion_id, log_path, line_count, settings.REDIS_HOST,
   settings.REDIS_PORT, settings.REDIS_PASSWD)
  # settings.TAIL_LOG_URL是tornado中MainHandler对应的url,把其它几个
  # 参数组合成最终的URL，直接访问这个URL就可以在浏览器中实时读取日志了。
  url = "{}&＃63;type=remote&log={}&hostname={}&cmd={}".format(
   settings.TAIL_LOG_URL, log_path, minion_id, cmd)
  # 这一步的操作确保同一个日志文件只有一个脚本在读取，避免日志信息重复，这一步
  # 也很重要，必不可少
  local.cmd(minion_id, "cmd.run",
     ["kill `ps aux|grep logtail.py|grep %s|grep -v grep|awk '{print $2}'`" % (log_path,)])
  return JsonResponse({"success": True, "url": url})

# encoding: utf-8
from __future__ import unicode_literals, division

import math
import time
import sys
import socket
import signal
import redis

FLAG = True


def get_last_lines(f, num=10):
 """读取文件的最后几行
 """
 size = 1000
 try:
  f.seek(-size, 2)
 except IOError: # 文件内容不足size
  f.seek(0)
  return f.readlines()[-num:]

 data = f.read()
 lines = data.splitlines()
 n = len(lines)
 while n