simAGV/helper/logger.py

""" 日志指定目录扫描以及切割, 有效文件名格式如下:
        包含一个.的日志文件: [^.]+.log
    其他忽略
"""

import os
import fnmatch
import shutil
import time
import datetime
import logging
import warnings
import requests

from logging import FileHandler, StreamHandler
from conf import setting

from .common import Singleton, str2int


MAXSIZE = 200 * 1024 * 1024


def check_and_rotate_logfile(dirname, logger, maxsize=10 * 1024 * 1024, maxtime=30 * 24 * 3600):
    """每周定时切换日志
    NOTE: 如果日志文件大小小于10MB, 则不进行切换操作
    """
    if not os.path.exists(dirname):
        logger.error(f'日志检查: 日志目录({dirname})不存在, 请检查')
        return

    # 1. 切割日志
    now = datetime.datetime.now()
    date_str = '{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}'.format(now.year, now.month, now.day, now.hour, now.minute, now.second)
    for filename in os.listdir(dirname):
        # a. 拷贝
        filepath = dirname + os.sep + filename
        if not os.path.isfile(filepath):
            continue
        if filename.count('.') > 1:
            continue
        if not fnmatch.fnmatchcase(filename, '*.log'):
            continue
        filesize = os.path.getsize(filepath)
        if filesize <= maxsize:  # 10MB以下文件不切割
            continue
        new_filepath = dirname + os.sep + filename.rsplit('.log', 1)[0] + '.' + date_str + '.log'
        shutil.copyfile(filepath, new_filepath)
        # b. 清空
        with open(filepath, 'r+', encoding='utf8') as fd:
            fd.seek(0)
            fd.truncate()

    # 2. 删除老日志
    bretime = time.time() - maxtime  # 30天前
    for filename in os.listdir(dirname):
        filepath = dirname + os.sep + filename
        if not os.path.isfile(filepath):
            continue
        if fnmatch.fnmatchcase(filename, '*.tar.gz'):  # 清理压缩包
            os.remove(filepath)
            continue
        if not fnmatch.fnmatchcase(filename, '*.log'):
            continue
        if filename.count('.') <= 1:
            continue
        try:
            if os.path.getmtime(filepath) <= bretime:
                os.remove(filepath)
            logger.exception(f'日志检查: 删除日志{filepath}成功')
        except Exception as e:
            logger.exception(f'日志检查: 删除日志{filepath}失败, msg: {e}')


def join_multiple_files(logger, filepaths, destfilepath, needsize, date_str):
    """join_multiple_files: 从filepaths中提取needsize大小的数据并append到destfilepath中

    :param filepaths: 按照时间从老到新的有序文件路径列表
    :param destfilepath: 目标文件
    :param needsize: 需要提取的数据大小
    :param date_str: 时间字符串
    """
    _min_size = 1024 * 1024
    for filepath in reversed(filepaths):
        # 获取源文件(截取拷贝等操作)
        need_delete = False
        filesize = os.path.getsize(filepath)
        if filesize == needsize:
            _tmp_filepath = filepath
            needsize = 0
        elif filesize < needsize:
            _tmp_filepath = filepath
            needsize = int((needsize - filesize) / _min_size)
        else:
            _tmp_filepath = filepath + f'.tmp{date_str}'
            skip_block = int((filesize - needsize) / _min_size)  # 跳过块, 没块大小1MB
            cmd = f'dd if={filepath} of={_tmp_filepath} skip={skip_block} ibs={_min_size} obs={_min_size}'
            needsize = 0
            val = os.system(cmd)
            if val:
                cmd = f'dd if={filepath} of={_tmp_filepath} skip={skip_block} ibs=1M obs=1M'
                val = os.system(cmd)
                if val:
                    msg = f'执行命令:{cmd}失败, code: {val}'
                    logger.error(msg)
                    return False, msg
            need_delete = True

        # 拼接和删除临时文件
        try:
            cmd = f'cat {_tmp_filepath} >> {destfilepath}'
            val = os.system(cmd)
            if val:
                msg = f'执行命令:{cmd}失败, code: {val}'
                logger.error(msg)
                return False, msg
        finally:
            if need_delete:
                os.remove(_tmp_filepath)

        # 判断needsize大小
        logger.info(f'成功拷贝文件:{filepath}中的内容到:{destfilepath}中')
        if needsize <= 0:
            break

    return True, destfilepath


def zip_and_transform_logfile(dirname, logger, fileprefix='yanei', minsize=MAXSIZE, tarfile=None):
    """zip_and_transform_logfile, 打包获取日志文件中最近20MB(传入参数)的数据并返回

    :param dirname: 日志目录
    :param logger: 日志记录对象
    :param fileprefix: 日志文件前缀
    :param minsize: 截取的日志文件大小
    :param tarfile: 如果该值存在, 则会

    NOTE: 该函数仅仅适用于Linux系统
    """
    if not os.path.exists(dirname):
        msg = f'日志检查: 日志目录({dirname})不存在, 请检查'
        logger.error(msg)
        return False, msg

    # 获取当前所有server.log日志以及自动备份的日志: yanei.20211026160000.log
    can_cutout_filepaths = []
    current_filename = f'{fileprefix}.log'  # 当前正在记录日志的文件
    current_filepath = dirname + os.sep + current_filename
    for filename in os.listdir(dirname):
        filepath = dirname + os.sep + filename
        if not os.path.isfile(filepath):
            continue
        if fnmatch.fnmatchcase(filename, '*.tar.gz'):
            continue
        if not fnmatch.fnmatchcase(filename, f'{fileprefix}*'):
            continue
        if filename == current_filename:
            continue
        can_cutout_filepaths.append(filepath)
    _can_cutout_filepaths = []

    for filepath in can_cutout_filepaths:
        try:
            x = str2int(filepath.split('.')[1:2][0])
            if not isinstance(x, int):
                api_logger.error(f'发现一个未知格式的文件:{filepath}, 忽略该文件')
                continue
        except Exception as msg:
            api_logger.error(f'发现一个未知格式的文件:{filepath}, 忽略该文件')
        else:
            _can_cutout_filepaths.append(filepath)
    can_cutout_filepaths = _can_cutout_filepaths
    can_cutout_filepaths = sorted(can_cutout_filepaths, key=lambda x: str2int(x.split('.')[1:2][0]))

    # 截取并拼接文件
    current_filesize = os.path.getsize(current_filepath)
    _min_size = 1024 * 1024
    now = datetime.datetime.now()
    date_str = '{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}'.format(now.year, now.month, now.day, now.hour, now.minute, now.second)
    save_filename = f'server.log.tmp.{date_str}'
    save_filepath = dirname + os.sep + save_filename
    if current_filesize > minsize:
        skip_block = int((current_filesize - minsize) / _min_size)  # 跳过块, 没块大小1MB
        cmd = f'dd if={current_filepath} of={save_filepath} skip={skip_block} ibs={_min_size} obs={_min_size}'
        val = os.system(cmd)
        if val:
            cmd = f'dd if={current_filepath} of={save_filepath} skip={skip_block} ibs=1M obs=1M'
            val = os.system(cmd)
            if val:
                msg = f'执行命令:{cmd}失败, code: {val}'
                logger.error(msg)
                return False, msg
        logger.info(f'日志文件足够大, 截取:{current_filepath}到{save_filepath}中成功')
    elif current_filesize == minsize:
        save_filepath = shutil.copyfile(current_filepath, save_filepath)
        logger.info(f'日志文件大小刚好, 拷贝:{current_filepath}到{save_filepath}中成功')
    else:
        save_filepath = shutil.copyfile(current_filepath, save_filepath)
        logger.info(f'日志文件大小较小, 先拷贝:{current_filepath}到{save_filepath}中成功')
        needsize = int((minsize - current_filesize) / _min_size)
        succ, msg = join_multiple_files(logger, can_cutout_filepaths, save_filepath, needsize, date_str)
        if not succ:
            return False, msg

    # 打包, 其中压缩包文件会在定时任务函数中清理
    tar_filename = f'YANEILog_.{date_str}.tar.gz' if not tarfile else tarfile
    tar_filepath = [dirname, tar_filename]
    try:
        cmd = f'cd {dirname} && tar zcf {tar_filename} {save_filename}'
        val = os.system(cmd)
        if val:
            msg = f'执行命令:{cmd}失败, code: {val}'
            logger.error(msg)
            return False, msg
    finally:
        os.remove(save_filepath)

    return True, tar_filepath


class YaneFilter(logging.Filter):
    """自定义logging过滤器, 通过redis或者mysql获取当前记录的最新日志level, 根据
    该level决定是否记录某一条日志
    """

    def filter(self, record):
        return super(YaneFilter, self).filter(record)


class YaneiErrorFilter(logging.Filter):
    """自由控制: 记录error以上的日志"""

    def filter(self, record):
        try:
            return record.levelno >= logging.ERROR
        except Exception:
            return super(YaneiErrorFilter, self).filter(record)


@Singleton
class YaneiLoggerGenerator:
    DEFAULT_LOG_FILE = 'server.log'  # 默认日志存储文件
    DEFAULT_ERROR_LOG_FILE = 'error.log'

    """ 日志logger生成工具
    NOTE: 非多进程安全, 并发量高的情况下可能造成日志丢失

    @默认格式: 日志等级 时间 日志所在文件名:日志所在行号 - 固定前缀 - 日志信息
    @生成logger或获取某一个logger:
        from app.common.log import g_logger_generator

        # 表示获取api日志对象, 日志会记录到api.log, 固定前缀为API
        app.apiLogger = g_logger_generator.get_logger('api', 'api.log', prefix='API')

        # 表示获取切换日志对象, 日志会记录到默认文件server.log, 固定前缀为SWITCH
        app.celeryLogger = g_logger_generator.get_logger('switch', prefix='SWITCH')

        # 表示获取文件同步日志对象, 日志会记录到默认文件server.log, 固定前缀为DELAY
        app.delayLogger = g_logger_generator.get_logger('delay', prefix='DELAY')
    """

    def __init__(self):
        self._loggers = {}  # 当前进程中logger集: name -> logger
        self.common_fmt_prefix = '%(levelname)s %(process)d:%(thread)d %(asctime)s %(filename)s:%(lineno)s'

        # 关闭requests, urllib3告警日志
        requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
        warnings.simplefilter('ignore', ResourceWarning)

    def _formatter(self, fmt, prefix):
        return logging.Formatter(fmt + f' - {prefix} - ' + '%(message)s')

    def update_file_mode(self, filepath, mode=0o666):
        if os.path.exists(filepath):
            status = os.stat(filepath)
            if (status.st_mode & mode) == mode:
                return

        old_umask = os.umask(0)
        with open(filepath, 'w', encoding='utf8') as fd:
            fd.write('.')
        os.chmod(filepath, mode)
        os.umask(old_umask)

    def get_logger(self, name, filename=None, level=logging.INFO, prefix='COMMON', stream=True):
        """get_logger, 获取name指定的日志句柄对象

        :param name:logger名称
        :param filename:日志存储的文件名
        :param level:日志记录等级
        :param prefix:日志记录前缀, 用以区分同一个日志文件中的不同日志信息
        :param stream:是否日志打印到stdout, 其中celery异步任务中的日志会自动重定向到标准输出中, 不需要重新设置
            如果本地开启调试, 则可以传递该值为True, 确保在本地开发时将日志打印到stdout

        @return: 返回一个{name}Logger的logger对象
        """

        if name in self._loggers:
            return self._loggers[name]

        if not os.path.exists('logs'):
            os.makedirs('logs')
        filename = self.DEFAULT_LOG_FILE if not filename else filename

        my_formatter = self._formatter(self.common_fmt_prefix, prefix)
        streamHandler = StreamHandler()
        streamHandler.setFormatter(my_formatter)
        _filename = f'logs{os.sep}{filename}'
        self.update_file_mode(os.path.join(setting.BASE_DIR, _filename))

        logHandler = FileHandler(_filename)
        logHandler.suffix = '%Y-%m-%d.%M-%S.log'
        logHandler.setFormatter(my_formatter)

        my_logger = logging.getLogger(name)
        if not my_logger.handlers:
            my_logger.addHandler(logHandler)
            my_logger.setLevel(level)
            if stream:
                my_logger.addHandler(streamHandler)

        self._loggers[name] = my_logger
        return my_logger

    def update_default_logger(self, logger, filename=None, level=logging.INFO, prefix='COMMON', clear=False, stream=True):
        """更新某一个logger的日志配置, 参数参考get_logger"""
        if logger.name not in self._loggers:
            self._loggers[logger.name] = logger

        filename = self.DEFAULT_LOG_FILE if not filename else filename

        # NOTE: 删除logger默认的handler, 如果需要则删除下面功能
        if clear:
            for old_handler in logger.handlers:
                logger.removeHandler(old_handler)

        my_formatter = self._formatter(self.common_fmt_prefix, prefix)
        streamHandler = StreamHandler()
        streamHandler.setFormatter(my_formatter)
        if stream:
            logger.addHandler(streamHandler)

        _filename = f'logs{os.sep}{filename}'
        self.update_file_mode(os.path.join(setting.BASE_DIR, _filename))
        logHandler = FileHandler(_filename)
        logHandler.suffix = '%Y-%m-%d.%M-%S.log'
        logHandler.setFormatter(my_formatter)
        logger.addHandler(logHandler)

        # 增加一个单独记录错误日志的handler(不需要重定向到标准输出)
        _filename = f'logs{os.sep}{self.DEFAULT_ERROR_LOG_FILE}'
        self.update_file_mode(os.path.join(setting.BASE_DIR, _filename))
        errorhandler = FileHandler(_filename)
        errorhandler.suffix = '%Y-%m-%d.%M-%S.log'
        errorhandler.setFormatter(my_formatter)
        errorhandler.addFilter(YaneiErrorFilter())
        logger.addHandler(errorhandler)

        logger.setLevel(logging.INFO)

        return logger


__g_logger_generator = YaneiLoggerGenerator()
api_logger = __g_logger_generator.get_logger('api', filename='api.log', level=setting.YANEI_LOG_LEVEL, prefix='API', stream=True)
logger = __g_logger_generator.get_logger('server', level=setting.YANEI_LOG_LEVEL, prefix='SERVER', stream=True)