Handle multiline log output without proper log file timestamp headers like produced by KnowIt errors

Author: JaiZed

bazarr/api/system/logs.py

@@ -54,9 +54,13 @@ def get(self):
             include = include.casefold()
             exclude = exclude.casefold()
 
+        # regular expression to identify the start of a log record (timestamp-based)
+        record_start_pattern = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+
         with io.open(get_log_file_path(), encoding='UTF-8') as file:
             raw_lines = file.read()
             lines = raw_lines.split('|\n')
+            multi_line_record = []
             for line in lines:
                 if line == '':
                     continue
@@ -86,18 +90,34 @@ def get(self):
                         skip = exclude in compare_line
                     if skip:
                         continue
-                raw_message = line.split('|')
-                raw_message_len = len(raw_message)
-                if raw_message_len > 3:
-                    log = dict()
-                    log["timestamp"] = raw_message[0]
-                    log["type"] = raw_message[1].rstrip()
-                    log["message"] = raw_message[3]
-                    if raw_message_len > 4 and raw_message[4] != '\n':
-                        log['exception'] = raw_message[4].strip('\'').replace('  ', '\u2003\u2003')
-                    else:
-                        log['exception'] = None
-                    logs.append(log)
+                # check if the line has a timestamp that matches the start of a new log record
+                if record_start_pattern.match(line):
+                    if multi_line_record:
+                        # finalize the multi line record and update the exception of the last entry 
+                        last_log = logs[-1]
+                        last_log["exception"] += "\n" + "\n".join(multi_line_record)
+                        # reset for the next multi-line record
+                        multi_line_record = []
+                    raw_message = line.split('|')
+                    raw_message_len = len(raw_message)
+                    if raw_message_len > 3:
+                        log = dict()
+                        log["timestamp"] = raw_message[0]
+                        log["type"] = raw_message[1].rstrip()
+                        log["message"] = raw_message[3]
+                        if raw_message_len > 4 and raw_message[4] != '\n':
+                            log['exception'] = raw_message[4].strip('\'').replace('  ', '\u2003\u2003')
+                        else:
+                            log['exception'] = None
+                        logs.append(log)
+                else:
+                    # accumulate lines that do not have new record header timestamps
+                    multi_line_record.append(line.strip())
+
+            if multi_line_record:
+                # finalize the multi line record and update the exception of the last entry 
+                last_log = logs[-1]
+                last_log["exception"] += "\n".join(multi_line_record)
 
             logs.reverse()
         return marshal(logs, self.get_response_model, envelope='data')