instrument application with prometheus and adding grafana + dashboards

pull/3/head
lance 4 years ago
parent 3636f6bbcd
commit 92515e7c1b

@ -16,3 +16,25 @@ services:
command: ["./manage.py", "run_huey", "-q"]
depends_on:
- "db"
grafana:
image: grafana/grafana:6.5.0
ports:
- 3000:3000
environment:
GF_SECURITY_ADMIN_USER: admin
HOSTNAME: grafana
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
volumes:
- ./Docker/files/grafana/dashboards.yaml:/etc/grafana/provisioning/dashboards/default.yaml:ro
- ./Docker/files/grafana/prometheus.yaml:/etc/grafana/provisioning/datasources/prometheus.yaml:ro
- ./Docker/files/grafana/xmrauctions_metrics.json:/var/lib/grafana/dashboards/xmrauctions_metrics.json:ro
- ./data/grafana:/var/lib/grafana
prometheus:
image: prom/prometheus:v2.15.2
ports:
- 9090:9090
volumes:
- ./Docker/files/prometheus:/etc/prometheus
- ./data/prometheus/

@ -0,0 +1,13 @@
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: true
editable: true
updateIntervalSeconds: 60
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

@ -0,0 +1,9 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus:9090
access: proxy
isDefault: true
timeInterval: 10s

@ -0,0 +1,642 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 4,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 19,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "total",
"sortDesc": true,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "GET - prometheus-django-metrics",
"hideTooltip": true,
"legend": false,
"lines": false
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(django_http_requests_total_by_view_transport_method_total[1m]) * 60",
"instant": false,
"legendFormat": "{{ method }} - {{ view }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Requests by Method/View",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 5,
"x": 19,
"y": 0
},
"hiddenSeries": false,
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Unapplied",
"color": "#F2495C"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "job:django_migrations_applied_total:max",
"legendFormat": "Applied",
"refId": "A"
},
{
"expr": "job:django_migrations_unapplied_total:max",
"legendFormat": "Unapplied",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "DB Migrations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"decimals": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 19,
"x": 0,
"y": 8
},
"hiddenSeries": false,
"id": 8,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "GET - prometheus-django-metrics",
"hideTooltip": true,
"legend": false
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(django_http_requests_latency_seconds_by_view_method_sum{job=\"django\"}[1m])",
"legendFormat": "{{ method }} - {{ view }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Request Latency",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 5,
"x": 19,
"y": 8
},
"hiddenSeries": false,
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "DB Connections",
"color": "#B877D9"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(django_db_new_connections_total[1m]) * 60",
"instant": false,
"legendFormat": "DB Connections",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "DB Connections",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "none",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"decimals": 2,
"description": "",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 19,
"x": 0,
"y": 16
},
"hiddenSeries": false,
"id": 10,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sideWidth": null,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(django_model_updates_total{job=\"django\"}[1m]) * 60",
"legendFormat": "{{ model }} Updates",
"refId": "C"
},
{
"expr": "rate(django_model_deletes_total{job=\"django\"}[1m]) * 60",
"legendFormat": "{{ model }} Deletes",
"refId": "B"
},
{
"expr": "rate(django_model_inserts_total{job=\"django\"}[1m]) * 60",
"legendFormat": "{{ model }} Inserts",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "DB Operations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "opm",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 5,
"x": 19,
"y": 16
},
"hiddenSeries": false,
"id": 12,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "django",
"color": "#5794F2"
},
{
"alias": "prometheus",
"color": "#FF9830"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(process_cpu_seconds_total[1m]) * 100",
"legendFormat": "{{ job }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Estimated CPU",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "30s",
"schemaVersion": 21,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "browser",
"title": "XMR Auctions",
"uid": "pVe331UWk",
"version": 27
}

@ -1,11 +0,0 @@
input {
http {}
}
output {
stdout { codec => "rubydebug" }
elasticsearch {
hosts => ["es01:9200"]
index => "logstash-%{+YYYY-MM-dd}"
}
}

@ -0,0 +1,105 @@
groups:
- name: django.rules
rules:
- record: job:django_http_requests_before_middlewares_total:sum_rate30s
expr: sum(rate(django_http_requests_before_middlewares_total[30s])) BY (job)
- record: job:django_http_requests_unknown_latency_total:sum_rate30s
expr: sum(rate(django_http_requests_unknown_latency_total[30s])) BY (job)
- record: job:django_http_ajax_requests_total:sum_rate30s
expr: sum(rate(django_http_ajax_requests_total[30s])) BY (job)
- record: job:django_http_responses_before_middlewares_total:sum_rate30s
expr: sum(rate(django_http_responses_before_middlewares_total[30s])) BY (job)
- record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s
expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s]))
BY (job)
- record: job:django_http_requests_body_total_bytes:sum_rate30s
expr: sum(rate(django_http_requests_body_total_bytes[30s])) BY (job)
- record: job:django_http_responses_streaming_total:sum_rate30s
expr: sum(rate(django_http_responses_streaming_total[30s])) BY (job)
- record: job:django_http_responses_body_total_bytes:sum_rate30s
expr: sum(rate(django_http_responses_body_total_bytes[30s])) BY (job)
- record: job:django_http_requests_total:sum_rate30s
expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job)
- record: job:django_http_requests_total_by_method:sum_rate30s
expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job, method)
- record: job:django_http_requests_total_by_transport:sum_rate30s
expr: sum(rate(django_http_requests_total_by_transport[30s])) BY (job, transport)
- record: job:django_http_requests_total_by_view:sum_rate30s
expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
view)
- record: job:django_http_requests_total_by_view_transport_method:sum_rate30s
expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
view, transport, method)
- record: job:django_http_responses_total_by_templatename:sum_rate30s
expr: sum(rate(django_http_responses_total_by_templatename[30s])) BY (job, templatename)
- record: job:django_http_responses_total_by_status:sum_rate30s
expr: sum(rate(django_http_responses_total_by_status[30s])) BY (job, status)
- record: job:django_http_responses_total_by_status_name_method:sum_rate30s
expr: sum(rate(django_http_responses_total_by_status_name_method[30s])) BY (job,
status, name, method)
- record: job:django_http_responses_total_by_charset:sum_rate30s
expr: sum(rate(django_http_responses_total_by_charset[30s])) BY (job, charset)
- record: job:django_http_exceptions_total_by_type:sum_rate30s
expr: sum(rate(django_http_exceptions_total_by_type[30s])) BY (job, type)
- record: job:django_http_exceptions_total_by_view:sum_rate30s
expr: sum(rate(django_http_exceptions_total_by_view[30s])) BY (job, view)
- record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "50"
- record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "95"
- record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "99"
- record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "99.9"
- record: job:django_http_requests_latency_seconds:quantile_rate30s
expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "50"
- record: job:django_http_requests_latency_seconds:quantile_rate30s
expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "95"
- record: job:django_http_requests_latency_seconds:quantile_rate30s
expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "99"
- record: job:django_http_requests_latency_seconds:quantile_rate30s
expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
BY (job, le))
labels:
quantile: "99.9"
- record: job:django_model_inserts_total:sum_rate1m
expr: sum(rate(django_model_inserts_total[1m])) BY (job, model)
- record: job:django_model_updates_total:sum_rate1m
expr: sum(rate(django_model_updates_total[1m])) BY (job, model)
- record: job:django_model_deletes_total:sum_rate1m
expr: sum(rate(django_model_deletes_total[1m])) BY (job, model)
- record: job:django_db_new_connections_total:sum_rate30s
expr: sum(rate(django_db_new_connections_total[30s])) BY (alias, vendor)
- record: job:django_db_new_connection_errors_total:sum_rate30s
expr: sum(rate(django_db_new_connection_errors_total[30s])) BY (alias, vendor)
- record: job:django_db_execute_total:sum_rate30s
expr: sum(rate(django_db_execute_total[30s])) BY (alias, vendor)
- record: job:django_db_execute_many_total:sum_rate30s
expr: sum(rate(django_db_execute_many_total[30s])) BY (alias, vendor)
- record: job:django_db_errors_total:sum_rate30s
expr: sum(rate(django_db_errors_total[30s])) BY (alias, vendor, type)
- record: job:django_migrations_applied_total:max
expr: max(django_migrations_applied_total) BY (job, connection)
- record: job:django_migrations_unapplied_total:max
expr: max(django_migrations_unapplied_total) BY (job, connection)

@ -0,0 +1,23 @@
global:
scrape_interval: 10s
evaluation_interval: 60s
external_labels:
monitor: django-monitor
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "django"
static_configs:
- targets: ["gunicorn:8000"]
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
rule_files:
- "django_rules.yml"

@ -1,10 +1,11 @@
from django_prometheus.models import ExportModelOperationsMixin
from django.db import models
from django.contrib.auth.models import User
from items.models import Item
from core.validators import address_is_valid_monero
class ItemBid(models.Model):
class ItemBid(ExportModelOperationsMixin('item_bid'), models.Model):
item = models.ForeignKey(Item, related_name='bids', on_delete=models.CASCADE)
bidder = models.ForeignKey(User, related_name='bidder', on_delete=models.CASCADE)
bid_date = models.DateTimeField(auto_now_add=True)

@ -1,8 +1,9 @@
from django_prometheus.models import ExportModelOperationsMixin
from django.db import models
from django.contrib.auth.models import User
class UserShippingAddress(models.Model):
class UserShippingAddress(ExportModelOperationsMixin('shipping_address'), models.Model):
user = models.ForeignKey(User, related_name='profile', on_delete=models.CASCADE)
address1 = models.CharField(max_length=100)
address2 = models.CharField(max_length=100, blank=True)

@ -1,5 +1,6 @@
from os import path as os_path
from secrets import token_urlsafe
from django_prometheus.models import ExportModelOperationsMixin
from django.db import models
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.contrib.auth.models import User
@ -10,7 +11,7 @@ from core.monero import AuctionDaemon
from core.validators import address_is_valid_monero
class Item(models.Model):
class Item(ExportModelOperationsMixin('item'), models.Model):
owner = models.ForeignKey(User, related_name='owner', on_delete=models.CASCADE)
name = models.CharField(max_length=100)
list_date = models.DateTimeField(auto_now_add=True)
@ -25,7 +26,7 @@ class Item(models.Model):
return f"{self.id} - {self.owner} - {self.name}"
class ItemImage(models.Model):
class ItemImage(ExportModelOperationsMixin('item_image'), models.Model):
item = models.ForeignKey(Item, related_name='images', on_delete=models.CASCADE)
image = models.ImageField(upload_to='%Y/%m/%d')
thumbnail = models.ImageField(upload_to='%Y/%m/%d')

@ -4,6 +4,7 @@ boto3==1.10.45
coverage==5.0.3
django-anymail[mailgun]==7.0.0
django-cors-headers==3.2.0
django-prometheus==2.0.0
django-redis==4.11.0
django-registration==3.0.1
django-storages==1.8.0

@ -1,3 +1,4 @@
from django_prometheus.models import ExportModelOperationsMixin
from django.db import models
from django.contrib.auth.models import User
from django.conf import settings
@ -5,7 +6,7 @@ from items.models import Item
from bids.models import ItemBid
class ItemSale(models.Model):
class ItemSale(ExportModelOperationsMixin('item_sale'), models.Model):
item = models.ForeignKey(Item, related_name='sales', on_delete=models.CASCADE)
bid = models.ForeignKey(ItemBid, related_name='bids', on_delete=models.CASCADE)
escrow_address = models.CharField(max_length=96)

@ -68,10 +68,12 @@ INSTALLED_APPS = [
'core.apps.CoreConfig',
'huey.contrib.djhuey',
'corsheaders',
'anymail'
'anymail',
'django_prometheus'
]
MIDDLEWARE = [
'django_prometheus.middleware.PrometheusBeforeMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
@ -79,7 +81,8 @@ MIDDLEWARE = [
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware'
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django_prometheus.middleware.PrometheusAfterMiddleware'
]
ROOT_URLCONF = 'xmrauctions.urls'
@ -111,7 +114,7 @@ WSGI_APPLICATION = 'xmrauctions.wsgi.application'
DATABASES = {
'default': {
'ENGINE': os.environ.get('DB_ENGINE', 'django.db.backends.postgresql'),
'ENGINE': os.environ.get('DB_ENGINE', 'django_prometheus.db.backends.postgresql'),
'NAME': os.environ['DB_NAME'],
'PASSWORD': os.environ['DB_PASS'],
'PORT': os.environ.get('DB_PORT', 5432),

@ -27,6 +27,7 @@ urlpatterns = [
path('bids/', include('bids.urls')),
path('sales/', include('sales.urls')),
path('items/', include('items.urls')),
path('', include('django_prometheus.urls'))
]
# Load local images if running locally