logging.py 8.34 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
__author__ = 'Daniel Scheffler'
3

4
5
import sys
import warnings
6
7
import logging
import os
8

9
10
from ..options.config import GMS_config as CFG

11
12
13
14
# try:
#     from StringIO import StringIO  # Python 2
# except ImportError:
#     from io import StringIO  # Python 3
15
16
17


class GMS_logger(logging.Logger):
18
19
20
    def __init__(self, name_logfile, fmt_suffix=None, path_logfile=None, log_level='INFO', append=True,
                 log_to_joblog=True):
        # type: (str, any, str, any, bool, bool) -> None
21
22
        """Returns a logging.logger instance pointing to the given logfile path.
        :param name_logfile:
23
        :param fmt_suffix:      if given, it will be included into log formatter
24
        :param path_logfile:    if no path is given, only a StreamHandler is created
25
        :param log_level:       the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';
26
                                default: 'INFO')
27
28
        :param append:          <bool> whether to append the log message to an existing logfile (1)
                                or to create a new logfile (0); default=1
29
        :param log_to_joblog:   whether to additionally log all messages to the logfile of the GMS job (default=1)
30
        """
31

32
33
34
        # private attributes
        self._captured_stream = ''

35
36
37
38
39
40
        self.name_logfile = name_logfile
        self.fmt_suffix = fmt_suffix
        self.path_logfile = path_logfile
        self.log_level = log_level
        self.appendMode = append

41
        super(GMS_logger, self).__init__(name_logfile)
42

43
        self.path_logfile = path_logfile
44
        self.formatter_fileH = logging.Formatter('%(asctime)s' + (' [%s]' % fmt_suffix if fmt_suffix else '') +
45
46
                                                 ' %(levelname)s:   %(message)s', datefmt='%Y/%m/%d %H:%M:%S')
        self.formatter_ConsoleH = logging.Formatter('%(asctime)s' + (' [%s]' % fmt_suffix if fmt_suffix else '') +
47
                                                    ':   %(message)s', datefmt='%Y/%m/%d %H:%M:%S')
48

49
        # set fileHandler
50
51
52
53
54
55
56
57
58
59
60
61
62
63
        if path_logfile:
            # create output directory
            while not os.path.isdir(os.path.dirname(path_logfile)):
                try:
                    os.makedirs(os.path.dirname(path_logfile))
                except OSError as e:
                    if e.errno != 17:
                        raise
                    else:
                        pass

            # create FileHandler
            fileHandler = logging.FileHandler(path_logfile, mode='a' if append else 'w')
            fileHandler.setFormatter(self.formatter_fileH)
64
            fileHandler.setLevel(log_level)
65
66
67
        else:
            fileHandler = None

68
69
70
71
72
73
74
75
76
77
        # set fileHandler for job logfile
        if log_to_joblog:
            # create FileHandler
            joblog_fileHandler = logging.FileHandler(os.path.join(CFG.path_job_logs, '%s.log' % CFG.ID),
                                                     mode='a' if append else 'w')
            joblog_fileHandler.setFormatter(self.formatter_fileH)
            joblog_fileHandler.setLevel(log_level)
        else:
            joblog_fileHandler = None

78
79
80
81
82
        # create StreamHandler # TODO add a StringIO handler
        # self.streamObj     = StringIO()
        # self.streamHandler = logging.StreamHandler(stream=self.streamObj)
        # self.streamHandler.setFormatter(formatter)
        # self.streamHandler.set_name('StringIO handler')
83

84
        # create ConsoleHandler for logging levels DEGUG and INFO -> logging to sys.stdout
85
        consoleHandler_out = logging.StreamHandler(stream=sys.stdout)  # by default it would go to sys.stderr
86
87
88
89
90
91
92
93
94
95
        consoleHandler_out.setFormatter(self.formatter_ConsoleH)
        consoleHandler_out.set_name('console handler stdout')
        consoleHandler_out.setLevel(log_level)
        consoleHandler_out.addFilter(LessThanFilter(logging.WARNING))

        # create ConsoleHandler for logging levels WARNING, ERROR, CRITICAL -> logging to sys.stderr
        consoleHandler_err = logging.StreamHandler(stream=sys.stderr)
        consoleHandler_err.setFormatter(self.formatter_ConsoleH)
        consoleHandler_err.setLevel(logging.WARNING)
        consoleHandler_err.set_name('console handler stderr')
96

97
        self.setLevel(log_level)
98

99
        if not self.handlers:
100
101
            if fileHandler:
                self.addHandler(fileHandler)
102
103
            if joblog_fileHandler:
                self.addHandler(joblog_fileHandler)
104
            # self.addHandler(self.streamHandler)
105
106
            self.addHandler(consoleHandler_out)
            self.addHandler(consoleHandler_err)
107

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
            #    if append:
            #        logfileHandler = logging.FileHandler(path_logfile, mode='a')
            #    else:
            #        logfileHandler = logging.FileHandler(path_logfile, mode='w')
            #    logfileHandler.setFormatter(formatter)
            #    logfileHandler.setLevel(logging.CRITICAL)
            #    consoleHandler_out = logging.StreamHandler()
            #    consoleHandler_out.setFormatter(formatter)
            #    consoleHandler_out.setLevel(logging.CRITICAL)
            #    # logger.setLevel(logging.DEBUG)
            #    if CPUs == 1:
            #        if not logger.handlers:
            #            logger.addHandler(logfileHandler)
            #            logger.addHandler(consoleHandler_out)
            #    else:
            #        logger.addHandler(logfileHandler)
            #        logger.addHandler(consoleHandler_out)
125

126
127
128
129
130
131
132
133
134
135
136
    def __getstate__(self):
        self.close()
        return self.__dict__

    def __setstate__(self, ObjDict):
        """Defines how the attributes of GMS object are unpickled."""
        self.__init__(ObjDict['name_logfile'], fmt_suffix=ObjDict['fmt_suffix'], path_logfile=ObjDict['path_logfile'],
                      log_level=ObjDict['log_level'], append=True)
        ObjDict = self.__dict__
        return ObjDict

137
138
139
140
141
142
143
144
145
    @property
    def captured_stream(self):
        if not self._captured_stream:
            self._captured_stream = self.streamObj.getvalue()

        return self._captured_stream

    @captured_stream.setter
    def captured_stream(self, string):
146
        assert isinstance(string, str), "'captured_stream' can only be set to a string. Got %s." % type(string)
147
148
        self._captured_stream = string

149
    def close(self):
150
        # update captured_stream and flush stream
151
152
        # self.captured_stream += self.streamObj.getvalue()
        # print(self.handlers[:])
153

154
155
156
        # self.removeHandler(self.streamHandler)
        # print(dir(self.streamHandler))
        # self.streamHandler = None
157
158

        for handler in self.handlers[:]:
159
            try:
160
161
162
                # if handler.get_name()=='StringIO handler':
                # self.streamObj.flush()
                # self.streamHandler.flush()
163

164
                handler.close()
165
                self.removeHandler(handler)  # if not called with '[:]' the StreamHandlers are left open
166
            except PermissionError:
167
                warnings.warn('Could not properly close logfile due to a PermissionError: %s' % sys.exc_info()[1])
168

169
        if self.handlers[:]:
170
            warnings.warn('Not all logging handlers could be closed. Remaining handlers: %s' % self.handlers[:])
171

172
            # print('sh', self.streamHandler)
173
174
175
176
177
178
179
180
181
182
183
184
185

    def view_logfile(self):
        with open(self.path_logfile) as inF:
            print(inF.read())


def close_logger(logger):
    if logger and hasattr(logger, 'handlers'):
        for handler in logger.handlers[:]:  # if not called with '[:]' the StreamHandlers are left open
            try:
                handler.close()
                logger.removeHandler(handler)
            except PermissionError:
186
                warnings.warn('Could not properly close logfile due to a PermissionError: %s' % sys.exc_info()[1])
187
188
189
190
191
192

        if logger.handlers[:]:
            warnings.warn('Not all logging handlers could be closed. Remaining handlers: %s' % logger.handlers[:])


def shutdown_loggers():
193
194
195
196
197
198
199
200
201
202
    logging.shutdown()


class LessThanFilter(logging.Filter):
    # http://stackoverflow.com/questions/2302315/how-can-info-and-debug-logging-message-be-sent-to-stdout-and-higher-level-messag
    def __init__(self, exclusive_maximum, name=""):
        super(LessThanFilter, self).__init__(name)
        self.max_level = exclusive_maximum

    def filter(self, record):
203
        # non-zero return means we log this message
204
        return True if record.levelno < self.max_level else False