Improve logging and setup error notifications via webhook (#1)

add more logging and setup error notifs via webhook to mattermost

app/app.py

@@ -3,6 +3,7 @@ import requests
 from logging.config import dictConfig
 from flask import render_template
 from app.factory import create_app
+from app.library.cache import cache
 
 
 app = create_app()
@@ -10,11 +11,55 @@ app = create_app()
 
 @app.errorhandler(requests.exceptions.ConnectionError)
 def request_connection_error(e):
+    key_name = 'request_connection_error'
+    data = cache.redis.get(key_name)
+    if not data:
+        app.logger.error(f'HTTP connection error: {e}')
+        cache.store_data(key_name, 10, 1)
+
     return render_template('error.html'), 500
 
+dictConfig({
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "default": {
+            "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
+        },
+        "access": {
+            "format": "%(message)s",
+        }
+    },
+    "handlers": {
+        "console": {
+            "level": "INFO",
+            "class": "logging.StreamHandler",
+            "formatter": "default",
+            "stream": "ext://sys.stdout",
+        },
+        "mattermost": {
+            "class": "app.factory.MattermostHandler",
+            "formatter": "default",
+            "level": "ERROR",
+        }
+    },
+    "loggers": {
+        "gunicorn.error": {
+            "handlers": ["console"] if app.debug else ["console", "mattermost"],
+            "level": "INFO",
+            "propagate": False,
+        },
+        "gunicorn.access": {
+            "handlers": ["console"] if app.debug else ["console"],
+            "level": "INFO",
+            "propagate": False,
+        }
+    },
+    "root": {
+        "level": "DEBUG" if app.debug else "INFO",
+        "handlers": ["console"] if app.debug else ["console", "mattermost"],
+    }
+})
 
 if __name__ == '__main__':
     app.run()
-    gunicorn_logger = logging.getLogger('gunicorn.error')
-    app.logger.handlers = gunicorn_logger.handlers
-    app.logger.setLevel(gunicorn_logger.level)

app/config.py

@@ -49,3 +49,9 @@ DO_DROPLET_IMAGE = getenv('DO_DROPLET_IMAGE', 'ubuntu-20-04-x64')
 DO_DROPLET_SIZE = getenv('DO_DROPLET_SIZE', 's-2vcpu-2gb')
 DO_DROPLET_STORAGE_GB = int(getenv('DO_DROPLET_STORAGE_GB', 110))
 DO_DOMAIN = getenv('DO_DOMAIN', SERVER_NAME)
+
+# MatterMost
+MM_ICON = getenv('MM_ICON', 'https://web.getmonero.org/press-kit/symbols/monero-symbol-480.png')
+MM_CHANNEL = getenv('MM_CHANNEL', '')
+MM_USERNAME = getenv('MM_USERNAME', 'xmrcannon-local')
+MM_ENDPOINT = getenv('MM_ENDPOINT', '')

app/factory.py

@@ -1,12 +1,19 @@
 import click
+from logging import Handler
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
+from app.library.mattermost import post_webhook
 from app import config
 
 
 db = SQLAlchemy()
 
 
+class MattermostHandler(Handler):
+    def emit(self, record):
+        return post_webhook(self.format(record))
+
+
 def setup_db(app: Flask):
     uri = 'postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}'.format(
         user=config.DB_USER,

app/library/mattermost.py

@@ -0,0 +1,24 @@
+from requests import post as r_post
+from json import dumps
+from flask import current_app
+from app import config
+
+
+def post_webhook(msg):
+    if config.MM_ENDPOINT:
+        try:
+            current_app.logger.info('Posting webhook')
+            if current_app.config["DEBUG"]:
+                msg = "[DEBUG] " + msg
+            data = {
+                "text": msg,
+                "channel": config.MM_CHANNEL,
+                "username": config.MM_USERNAME,
+                "icon_url": config.MM_ICON
+            }
+            res = r_post(config.MM_ENDPOINT, data=dumps(data))
+            res.raise_for_status()
+            return True
+        except Exception as e:
+            current_app.logger.info(f'Unable to post webhook: {e}')
+            return False

bin/prod_app

@@ -20,6 +20,7 @@ gunicorn \
   --log-file $BASE/gunicorn.log \
   --pid $BASE/gunicorn.pid \
   --access-logfile $BASE/access.log \
+  --capture-output \
   --reload
 
 sleep 2

env-example

@@ -31,3 +31,8 @@ DO_DROPLET_IMAGE=ubuntu-20-04-x64
 DO_DROPLET_SIZE=s-2vcpu-2gb
 DO_DROPLET_STORAGE_GB=120
 DO_DOMAIN=__yourxmrcannon.net__
+
+MM_ICON=https://xxx.com/logo.png
+MM_CHANNEL=logs
+MM_USERNAME=xmrcannon-local
+MM_ENDPOINT=https://yourthing.cloud.mattermost.com/hooks/zzzzz