commit 752f048d99c57415982e7d795c8c79efc1dffd0c
parent 583c9e1b78e6ec0e933b9ee9d53746954f63842f
Author: Brian C. Lane <bcl@brianlane.com>
Date: Sat, 5 Aug 2017 16:50:44 -0700
Add structlog logging. Defaults to /var/tmp/strix.log
Diffstat:
5 files changed, 105 insertions(+), 23 deletions(-)
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,5 @@
-mypy
-pytest
bottle
+mypy
Pillow
+pytest
+structlog
diff --git a/src/strix/__init__.py b/src/strix/__init__.py
@@ -14,6 +14,7 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import multiprocessing as mp
import os
import re
import time
@@ -21,6 +22,7 @@ import threading
from . import cmdline
from . import queue
+from . import logger
from . import motion
## Check the motion args
@@ -99,6 +101,16 @@ def run() -> bool:
list(map(p_e, errors))
return False
+ # Start logging thread
+ logging_queue = mp.Queue()
+ logging_quit = threading.Event()
+ logging_thread = threading.Thread(name="logging-thread",
+ target=logger.listener,
+ args=(logging_queue, logging_quit, opts.log))
+ logging_thread.start()
+ running_threads = [(logging_thread, logging_quit)]
+
+ # Start queue monitor and processing thread (starts its own Multiprocessing threads)
queue_path = os.path.abspath(os.path.join(base_dir, "queue/"))
if not os.path.exists(queue_path):
print("ERROR: %s does not exist. Is motion running?" % queue_path)
@@ -106,9 +118,14 @@ def run() -> bool:
queue_quit = threading.Event()
queue_thread = threading.Thread(name="queue-thread",
target=queue.monitor_queue,
- args=(base_dir,queue_quit))
+ args=(logging_queue, base_dir, queue_quit))
queue_thread.start()
+ running_threads += [(queue_thread, queue_quit)]
+
+ # Start API thread (may start its own threads to handle requests)
+ # TODO
+ # Wait until it is told to exit
try:
while True:
time.sleep(10)
@@ -118,12 +135,12 @@ def run() -> bool:
print("Exiting due to ^C")
# Tell the threads to quit
- for event in [queue_quit]:
+ for _thread, event in running_threads:
event.set()
# Wait until everything is done
print("Waiting for threads to quit")
- for thread in [queue_thread]:
+ for thread, _event in running_threads:
thread.join()
return True
diff --git a/src/strix/logger.py b/src/strix/logger.py
@@ -0,0 +1,59 @@
+# logger.py
+#
+# Copyright (C) 2017 Brian C. Lane
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import sys
+import logging
+from logging.handlers import RotatingFileHandler, QueueListener, QueueHandler
+import multiprocessing as mp
+import threading
+
+import structlog
+
+
+structlog.configure(
+ processors=[
+ structlog.stdlib.filter_by_level,
+ structlog.stdlib.add_logger_name,
+ structlog.stdlib.add_log_level,
+ structlog.stdlib.PositionalArgumentsFormatter(),
+ structlog.processors.StackInfoRenderer(),
+ structlog.processors.format_exc_info,
+ structlog.processors.TimeStamper(fmt="iso"),
+ structlog.processors.UnicodeDecoder(),
+ structlog.processors.JSONRenderer(),
+ ],
+ context_class=dict,
+ logger_factory=structlog.stdlib.LoggerFactory(),
+ wrapper_class=structlog.stdlib.BoundLogger,
+ cache_logger_on_first_use=True,
+)
+
+def listener(queue: mp.Queue, stop_event: threading.Event, log_path: str) -> None:
+ handler = RotatingFileHandler(log_path, maxBytes=100*1024**2, backupCount=10)
+ formatter = logging.Formatter('%(message)s')
+ handler.setFormatter(formatter)
+ queue_listener = QueueListener(queue, handler) # type: ignore
+ queue_listener.start()
+ stop_event.wait()
+ queue_listener.stop()
+
+
+def log(queue: mp.Queue) -> structlog.BoundLogger:
+ handler = QueueHandler(queue) # type: ignore
+ root = structlog.get_logger()
+ root.addHandler(handler)
+ root.setLevel(logging.DEBUG)
+ return root
diff --git a/src/strix/queue.py b/src/strix/queue.py
@@ -23,28 +23,30 @@ import time
import threading
from PIL import Image
+import structlog
from typing import List
+from . import logger
+
THUMBNAIL_SIZE = (640, 480)
## Handle watching the queue and dispatching movie creation and directory moving
-def process_event(base_dir: str, event: str) -> None:
- print("Processing %s" % event)
- print("base_dir = %s" % base_dir)
+def process_event(log: structlog.BoundLogger, base_dir: str, event: str) -> None:
+ log.info(event_path=event, base_dir=base_dir)
# The actual path is the event with _ replaced by /
event_path = os.path.join(base_dir, event.replace("_", os.path.sep))
if not os.path.isdir(event_path):
- print("ERROR: event_path '%s' doesn't exist" % event_path)
+ log.error("event_path doesn't exist", event_path=event_path)
return
debug_path = os.path.join(event_path, "debug")
try:
os.mkdir(debug_path, mode=0o755)
except Exception as e:
- print("ERROR: Failed to create debug directory: %s" % e)
+ log.error("Failed to create debug directory", exception=str(e))
return
# Move the debug images into ./debug/
@@ -52,7 +54,7 @@ def process_event(base_dir: str, event: str) -> None:
for debug_img in glob(os.path.join(event_path, "*m.jpg")):
shutil.move(debug_img, debug_path)
except Exception as e:
- print("ERROR: Failed to move debug images")
+ log.debug("Failed to move debug images into ./debug/")
ffmpeg_cmd = ["ffmpeg", "-f", "image2", "-pattern_type", "glob", "-r", "10", "-i", "*.jpg", "-c:v",
"libvpx", "-crf", "10", "-b:v", "2M", "video.webm"]
@@ -61,13 +63,13 @@ def process_event(base_dir: str, event: str) -> None:
try:
subprocess.run(ffmpeg_cmd, cwd=event_path, check=True)
except Exception as e:
- print("ERROR: Failed to create video: %s" % e)
+ log.error("Failed to create video", exception=str(e))
# Make a movie out of the debug jpg images with ffmpeg
try:
subprocess.run(ffmpeg_cmd, cwd=debug_path, check=True)
except Exception as e:
- print("ERROR: Failed to create debug video: %s" % e)
+ log.error("Failed to create debug video", exception=str(e))
# Create a thumbnail of the middle image of the capture, on the theory that it
# has the best chance of being 'interesting'.
@@ -79,7 +81,7 @@ def process_event(base_dir: str, event: str) -> None:
im.thumbnail(THUMBNAIL_SIZE)
im.save(os.path.join(event_path, "thumbnail.jpg"), "JPEG")
except Exception as e:
- print("ERROR: Failed to create thumbnail: %s" % e)
+ log.error("Failed to create thumbnail", exception=str(e))
# Move the directory to its final location
try:
@@ -88,16 +90,18 @@ def process_event(base_dir: str, event: str) -> None:
first_time = first_jpg.rsplit("-", 1)[0]
event_path_base = os.path.split(event_path)[0]
dest_path = os.path.join(event_path_base, first_time)
- print("INFO: Destination path is %s" % dest_path)
+ log.info("Moved event to final location", dest_path=dest_path)
if not os.path.exists(dest_path):
os.rename(event_path, dest_path)
except Exception as e:
- print("ERROR: Moving %s to destination failed: %s" % (event_path, e))
+ log.error("Moving to destination failed", event_path=event_path, exception=str(e))
-def monitor_queue(base_dir: str, quit: threading.Event) -> None:
+def monitor_queue(logging_queue: mp.Queue, base_dir: str, quit: threading.Event) -> None:
threads = [] # type: List[mp.Process]
+ log = logger.log(logging_queue)
queue_path = os.path.abspath(os.path.join(base_dir, "queue/"))
+ log.info("Started queue monitor", queue_path=queue_path)
while not quit.is_set():
time.sleep(5)
# Remove any threads from the list that have finished
@@ -105,16 +109,16 @@ def monitor_queue(base_dir: str, quit: threading.Event) -> None:
if not t.is_alive():
threads.remove(t)
- print("Checking %s" % queue_path)
+ log.debug("queue check", queue_path=queue_path)
for event_file in glob(os.path.join(queue_path, "*")):
os.unlink(event_file)
event = os.path.split(event_file)[-1]
- thread = mp.Process(target=process_event, args=(base_dir, event))
+ thread = mp.Process(target=process_event, args=(log, base_dir, event))
threads.append(thread)
thread.start()
- print("monitor_queue waiting for threads to finish")
+ log.info("monitor_queue waiting for threads to finish")
for t in threads:
t.join()
- print("monitor_queue is quitting")
+ log.info("monitor_queue is quitting")
diff --git a/tox.ini b/tox.ini
@@ -3,13 +3,14 @@ envlist = py35
[testenv]
deps=
+ bottle
coverage
+ mypy
nose
pylint
pytest
- mypy
- bottle
Pillow
+ structlog
commands=
mypy --strict --ignore-missing-imports src/strix/ src/bin/strix
pylint --rcfile=pylint.rc -E src/strix/ src/bin/strix
