From 2567db144f2898fc40d27c6e3f7c25ebdf673ac8 Mon Sep 17 00:00:00 2001 From: lza_menace Date: Wed, 30 Dec 2020 22:49:32 +0000 Subject: [PATCH] Event logging/capture (#3) allow longer event type strings in db add graf container w/ pre-built dashboard remove elasticsearch and use db based event logging add init command Co-authored-by: lza_menace Reviewed-on: https://git.wownero.com/lza_menace/wowstash/pulls/3 --- Dockerfile | 9 - docker-compose.es.yaml | 22 -- docker-compose.yaml | 19 ++ files/dashboards.yaml | 13 ++ files/wowstash_ops.json | 311 +++++++++++++++++++++++++++ wowstash/blueprints/auth/routes.py | 14 +- wowstash/blueprints/wallet/routes.py | 17 +- wowstash/config.example.py | 1 - wowstash/factory.py | 5 + wowstash/library/docker.py | 3 - wowstash/library/elasticsearch.py | 22 -- wowstash/library/helpers.py | 12 ++ wowstash/models.py | 12 ++ 13 files changed, 389 insertions(+), 71 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.es.yaml create mode 100644 files/dashboards.yaml create mode 100644 files/wowstash_ops.json delete mode 100644 wowstash/library/elasticsearch.py create mode 100644 wowstash/library/helpers.py diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 605fa24..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:19.10 -WORKDIR /srv -COPY requirements.txt . -RUN apt-get update && apt-get install python3-pip -y -RUN python3 -m pip install -r requirements.txt -COPY wowstash wowstash/ -COPY bin/ bin/ -EXPOSE 4001 -CMD ["/srv/bin/prod-container"] diff --git a/docker-compose.es.yaml b/docker-compose.es.yaml deleted file mode 100644 index 9e34b45..0000000 --- a/docker-compose.es.yaml +++ /dev/null @@ -1,22 +0,0 @@ -services: - kibana: - image: docker.elastic.co/kibana/kibana:7.1.0 - ports: - - 5601:5601 - environment: - ELASTICSEARCH_HOSTS: http://elasticsearch:9200 - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0 - environment: - - discovery.type=single-node - - node.name=elasticsearch - - cluster.name=es-docker-cluster - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - ./data/elasticsearch:/usr/share/elasticsearch/data - ports: - - 9200:9200 diff --git a/docker-compose.yaml b/docker-compose.yaml index f482f6d..070328f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,3 +18,22 @@ services: container_name: wowstash_cache ports: - 6379:6379 + grafana: + image: grafana/grafana:6.5.0 + container_name: grafana + restart: unless-stopped + ports: + - 127.0.0.1:3001:3000 + environment: + HOSTNAME: grafana + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD} + GF_SERVER_ROOT_URL: ${GRAFANA_URL} + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_USERS_ALLOW_ORG_CREATE: "false" + volumes: + - ./files/dashboards.yaml:/etc/grafana/provisioning/dashboards/default.yaml:ro + - ./files/wowstash_ops.json:/var/lib/grafana/dashboards/wowstash_ops.json:ro + - grafana:/var/lib/grafana diff --git a/files/dashboards.yaml b/files/dashboards.yaml new file mode 100644 index 0000000..ce46f5a --- /dev/null +++ b/files/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: +- name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: true + editable: true + updateIntervalSeconds: 60 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/files/wowstash_ops.json b/files/wowstash_ops.json new file mode 100644 index 0000000..d30c5bc --- /dev/null +++ b/files/wowstash_ops.json @@ -0,0 +1,311 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [ + { + "params": [ + "$__interval", + "none" + ], + "type": "time" + } + ], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n $__timeGroupAlias(register_date,$__interval),\n avg(id) AS \"id\"\nFROM users\nWHERE\n $__timeFilter(register_date)\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + }, + { + "params": [ + "avg" + ], + "type": "aggregate" + }, + { + "params": [ + "id" + ], + "type": "alias" + } + ] + ], + "table": "users", + "timeColumn": "register_date", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "User Registrations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [ + { + "params": [ + "$__interval", + "0" + ], + "type": "time" + } + ], + "metricColumn": "type", + "rawQuery": false, + "rawSql": "SELECT\n $__timeGroupAlias(date,$__interval,0),\n type AS metric,\n count(\"user\") AS \"id\"\nFROM events\nWHERE\n $__timeFilter(date)\nGROUP BY 1,2\nORDER BY 1,2", + "refId": "A", + "select": [ + [ + { + "params": [ + "\"user\"" + ], + "type": "column" + }, + { + "params": [ + "count" + ], + "type": "aggregate" + }, + { + "params": [ + "id" + ], + "type": "alias" + } + ] + ], + "table": "events", + "timeColumn": "date", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Event Activity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Wowstash Ops", + "uid": "zvTlfCbGz", + "version": 1 +} diff --git a/wowstash/blueprints/auth/routes.py b/wowstash/blueprints/auth/routes.py index b7595df..6f23b6d 100644 --- a/wowstash/blueprints/auth/routes.py +++ b/wowstash/blueprints/auth/routes.py @@ -7,7 +7,7 @@ from wowstash.forms import Register, Login, Delete from wowstash.models import User from wowstash.factory import db, bcrypt from wowstash.library.docker import docker -from wowstash.library.elasticsearch import send_es +from wowstash.library.helpers import capture_event @auth_bp.route("/register", methods=["GET", "POST"]) @@ -33,7 +33,7 @@ def register(): db.session.commit() # Capture event, login user and redirect to wallet page - send_es({'type': 'register', 'user': user.email}) + capture_event(user.id, 'register') login_user(user) return redirect(url_for('wallet.setup')) @@ -63,7 +63,7 @@ def login(): return redirect(url_for('auth.login')) # Capture event, login user, and redirect to wallet page - send_es({'type': 'login', 'user': user.email}) + capture_event(user.id, 'login') login_user(user) return redirect(url_for('wallet.dashboard')) @@ -73,9 +73,9 @@ def login(): def logout(): if current_user.is_authenticated: docker.stop_container(current_user.wallet_container) - send_es({'type': 'stop_container', 'user': current_user.email}) + capture_event(current_user.id, 'stop_container') current_user.clear_wallet_data() - send_es({'type': 'logout', 'user': current_user.email}) + capture_event(current_user.id, 'logout') logout_user() return redirect(url_for('meta.index')) @@ -85,10 +85,10 @@ def delete(): form = Delete() if form.validate_on_submit(): docker.stop_container(current_user.wallet_container) - send_es({'type': 'stop_container', 'user': current_user.email}) + capture_event(current_user.id, 'stop_container') sleep(1) docker.delete_wallet_data(current_user.id) - send_es({'type': 'delete_wallet', 'user': current_user.email}) + capture_event(current_user.id, 'delete_wallet') current_user.clear_wallet_data(reset_password=True, reset_wallet=True) flash('Successfully deleted wallet data') return redirect(url_for('wallet.setup')) diff --git a/wowstash/blueprints/wallet/routes.py b/wowstash/blueprints/wallet/routes.py index e791287..ea60ed4 100644 --- a/wowstash/blueprints/wallet/routes.py +++ b/wowstash/blueprints/wallet/routes.py @@ -10,7 +10,7 @@ from socket import socket from datetime import datetime from wowstash.blueprints.wallet import wallet_bp from wowstash.library.docker import docker -from wowstash.library.elasticsearch import send_es +from wowstash.library.helpers import capture_event from wowstash.library.jsonrpc import Wallet, to_atomic from wowstash.library.cache import cache from wowstash.forms import Send, Delete, Restore @@ -29,6 +29,7 @@ def setup(): if restore_form.validate_on_submit(): c = docker.create_wallet(current_user.id, restore_form.seed.data) cache.store_data(f'init_wallet_{current_user.id}', 30, c) + capture_event(current_user.id, 'restore_wallet') current_user.wallet_created = True db.session.commit() return redirect(url_for('wallet.loading')) @@ -81,7 +82,7 @@ def dashboard(): seed = wallet.seed() spend_key = wallet.spend_key() view_key = wallet.view_key() - send_es({'type': 'load_dashboard', 'user': current_user.email}) + capture_event(current_user.id, 'load_dashboard') return render_template( 'wallet/dashboard.html', transfers=all_transfers, @@ -115,6 +116,7 @@ def connect(): current_user.wallet_container = wallet current_user.wallet_start = datetime.utcnow() db.session.commit() + capture_event(current_user.id, 'start_wallet') data = { 'result': 'success', 'message': 'Wallet has been connected' @@ -133,6 +135,7 @@ def create(): if current_user.wallet_created is False: c = docker.create_wallet(current_user.id) cache.store_data(f'init_wallet_{current_user.id}', 30, c) + capture_event(current_user.id, 'create_wallet') current_user.wallet_created = True db.session.commit() return redirect(url_for('wallet.loading')) @@ -173,13 +176,13 @@ def send(): # Check if Wownero wallet is available if wallet.connected is False: flash('Wallet RPC interface is unavailable at this time. Try again later.') - send_es({'type': 'tx_fail_rpc_unavailable', 'user': user.email}) + capture_event(user.id, 'tx_fail_rpc_unavailable') return redirect(redirect_url) # Quick n dirty check to see if address is WOW if len(address) not in [97, 108]: flash('Invalid Wownero address provided.') - send_es({'type': 'tx_fail_address_invalid', 'user': user.email}) + capture_event(user.id, 'tx_fail_address_invalid') return redirect(redirect_url) # Check if we're sweeping or not @@ -191,7 +194,7 @@ def send(): amount = to_atomic(Decimal(send_form.amount.data)) except: flash('Invalid Wownero amount specified.') - send_es({'type': 'tx_fail_amount_invalid', 'user': user.email}) + capture_event(user.id, 'tx_fail_amount_invalid') return redirect(redirect_url) # Send transfer @@ -202,10 +205,10 @@ def send(): msg = tx['message'].capitalize() msg_lower = tx['message'].replace(' ', '_').lower() flash(f'There was a problem sending the transaction: {msg}') - send_es({'type': f'tx_fail_{msg_lower}', 'user': user.email}) + capture_event(user.id, f'tx_fail_{msg_lower}') else: flash('Successfully sent transfer.') - send_es({'type': 'tx_success', 'user': user.email}) + capture_event(user.id, 'tx_success') return redirect(redirect_url) else: diff --git a/wowstash/config.example.py b/wowstash/config.example.py index 07131c7..99ccb23 100644 --- a/wowstash/config.example.py +++ b/wowstash/config.example.py @@ -37,7 +37,6 @@ DB_PASS = 'zzzzzzzzz' # Development TEMPLATES_AUTO_RELOAD = True -ELASTICSEARCH_ENABLED = False # Social SOCIAL = { diff --git a/wowstash/factory.py b/wowstash/factory.py index e1b2def..f51c5bf 100644 --- a/wowstash/factory.py +++ b/wowstash/factory.py @@ -77,6 +77,11 @@ def create_app(): user.clear_wallet_data() print(f'Wallet data cleared for user {user.id}') + @app.cli.command('init') + def init(): + import wowstash.models + db.create_all() + # Routes/blueprints from wowstash.blueprints.auth import auth_bp from wowstash.blueprints.wallet import wallet_bp diff --git a/wowstash/library/docker.py b/wowstash/library/docker.py index 05a2b6e..a43185c 100644 --- a/wowstash/library/docker.py +++ b/wowstash/library/docker.py @@ -9,7 +9,6 @@ from wowstash import config from wowstash.models import User from wowstash.factory import db from wowstash.library.jsonrpc import daemon -from wowstash.library.elasticsearch import send_es class Docker(object): @@ -66,7 +65,6 @@ class Docker(object): } } ) - send_es({'type': f'init_wallet', 'user': u.email}) return container.short_id def start_wallet(self, user_id): @@ -103,7 +101,6 @@ class Docker(object): } } ) - send_es({'type': 'start_wallet', 'user': u.email}) return container.short_id except APIError as e: if str(e).startswith('409'): diff --git a/wowstash/library/elasticsearch.py b/wowstash/library/elasticsearch.py deleted file mode 100644 index 5e62e39..0000000 --- a/wowstash/library/elasticsearch.py +++ /dev/null @@ -1,22 +0,0 @@ -from datetime import datetime -from elasticsearch import Elasticsearch -from wowstash import config - - -def send_es(data): - if getattr(config, 'ELASTICSEARCH_ENABLED', False): - try: - es = Elasticsearch( - [getattr(config, 'ELASTICSEARCH_HOST', 'localhost')] - ) - now = datetime.utcnow() - index_ts = now.strftime('%Y%m%d') - data['datetime'] = now - es.index( - index="{}-{}".format( - getattr(config, 'ELASTICSEARCH_INDEX_NAME', 'wowstash'), - index_ts - ), body=data) - except Exception as e: - print('Could not capture event in Elasticsearch: ', e) - pass # I don't really care if this logs... diff --git a/wowstash/library/helpers.py b/wowstash/library/helpers.py new file mode 100644 index 0000000..8dc2e42 --- /dev/null +++ b/wowstash/library/helpers.py @@ -0,0 +1,12 @@ +from wowstash.models import Event +from wowstash.factory import db + + +def capture_event(user_id, event_type): + event = Event( + user=user_id, + type=event_type + ) + db.session.add(event) + db.session.commit() + return diff --git a/wowstash/models.py b/wowstash/models.py index 2d87caf..4582b64 100644 --- a/wowstash/models.py +++ b/wowstash/models.py @@ -53,3 +53,15 @@ class User(db.Model): def __repr__(self): return self.email + + +class Event(db.Model): + __tablename__ = 'events' + + id = db.Column(db.Integer, primary_key=True) + type = db.Column(db.String(60)) + user = db.Column(db.Integer, db.ForeignKey(User.id)) + date = db.Column(db.DateTime, server_default=func.now()) + + def __repr__(self): + return self.id