|
|
@ -5,6 +5,7 @@ from time import sleep
|
|
|
|
import geoip2.database
|
|
|
|
import geoip2.database
|
|
|
|
import arrow
|
|
|
|
import arrow
|
|
|
|
import requests
|
|
|
|
import requests
|
|
|
|
|
|
|
|
import click
|
|
|
|
from flask import Blueprint
|
|
|
|
from flask import Blueprint
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
|
|
|
|
|
@ -22,57 +23,71 @@ def init():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@bp.cli.command("check")
|
|
|
|
@bp.cli.command("check")
|
|
|
|
def check():
|
|
|
|
def check_nodes():
|
|
|
|
diff = datetime.utcnow() - timedelta(hours=24)
|
|
|
|
diff = datetime.utcnow() - timedelta(hours=72)
|
|
|
|
checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
|
|
|
|
checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
|
|
|
|
for check in checks:
|
|
|
|
for check in checks:
|
|
|
|
print("Deleting check", check.id)
|
|
|
|
print("Deleting check", check.id)
|
|
|
|
check.delete_instance()
|
|
|
|
check.delete_instance()
|
|
|
|
nodes = Node.select().where(Node.validated == True)
|
|
|
|
nodes = Node.select().where(Node.validated == True)
|
|
|
|
for node in nodes:
|
|
|
|
for node in nodes:
|
|
|
|
now = datetime.utcnow()
|
|
|
|
|
|
|
|
hc = HealthCheck(node=node)
|
|
|
|
|
|
|
|
logging.info(f"Attempting to check {node.url}")
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
r = make_request(node.url)
|
|
|
|
check_node(node.url)
|
|
|
|
assert "status" in r.json()
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
assert "offline" in r.json()
|
|
|
|
exit()
|
|
|
|
assert "height" in r.json()
|
|
|
|
|
|
|
|
if 'donation_address' in r.json():
|
|
|
|
|
|
|
|
node.donation_address = r.json()['donation_address']
|
|
|
|
def check_node(_node):
|
|
|
|
has_cors = "Access-Control-Allow-Origin" in r.headers
|
|
|
|
if _node.startswith("http"):
|
|
|
|
is_ssl = node.url.startswith("https://")
|
|
|
|
node = Node.select().where(Node.url == _node).first()
|
|
|
|
if r.json()["status"] == "OK":
|
|
|
|
else:
|
|
|
|
node.web_compatible = has_cors and is_ssl
|
|
|
|
node = Node.select().where(Node.id == _node).first()
|
|
|
|
node.last_height = r.json()["height"]
|
|
|
|
if not node:
|
|
|
|
hc.health = True
|
|
|
|
print('node found')
|
|
|
|
highest_block = get_highest_block(node.nettype, node.crypto)
|
|
|
|
pass
|
|
|
|
healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
|
|
|
|
now = datetime.utcnow()
|
|
|
|
if r.json()["height"] < healthy_block:
|
|
|
|
hc = HealthCheck(node=node)
|
|
|
|
node.available = False
|
|
|
|
logging.info(f"Attempting to check {node.url}")
|
|
|
|
logging.info("unhealthy")
|
|
|
|
try:
|
|
|
|
else:
|
|
|
|
r = make_request(node.url)
|
|
|
|
node.available = True
|
|
|
|
assert "status" in r.json()
|
|
|
|
logging.info("success")
|
|
|
|
assert "offline" in r.json()
|
|
|
|
|
|
|
|
assert "height" in r.json()
|
|
|
|
|
|
|
|
if 'donation_address' in r.json():
|
|
|
|
|
|
|
|
node.donation_address = r.json()['donation_address']
|
|
|
|
|
|
|
|
has_cors = "Access-Control-Allow-Origin" in r.headers
|
|
|
|
|
|
|
|
is_ssl = node.url.startswith("https://")
|
|
|
|
|
|
|
|
if r.json()["status"] == "OK":
|
|
|
|
|
|
|
|
node.web_compatible = has_cors and is_ssl
|
|
|
|
|
|
|
|
node.last_height = r.json()["height"]
|
|
|
|
|
|
|
|
hc.health = True
|
|
|
|
|
|
|
|
highest_block = get_highest_block(node.nettype, node.crypto)
|
|
|
|
|
|
|
|
healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
|
|
|
|
|
|
|
|
if r.json()["height"] < healthy_block:
|
|
|
|
|
|
|
|
node.available = False
|
|
|
|
|
|
|
|
logging.info("unhealthy")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise
|
|
|
|
node.available = True
|
|
|
|
except:
|
|
|
|
logging.info("success")
|
|
|
|
logging.info("fail")
|
|
|
|
else:
|
|
|
|
node.datetime_failed = now
|
|
|
|
raise
|
|
|
|
node.available = False
|
|
|
|
except:
|
|
|
|
hc.health = False
|
|
|
|
logging.info("fail")
|
|
|
|
finally:
|
|
|
|
node.datetime_failed = now
|
|
|
|
node.datetime_checked = now
|
|
|
|
node.available = False
|
|
|
|
node.save()
|
|
|
|
hc.health = False
|
|
|
|
hc.save()
|
|
|
|
finally:
|
|
|
|
if (
|
|
|
|
node.datetime_checked = now
|
|
|
|
node.get_failed_checks().count() == node.get_all_checks().count()
|
|
|
|
node.save()
|
|
|
|
and node.get_all_checks().count() > 5
|
|
|
|
hc.save()
|
|
|
|
):
|
|
|
|
if (
|
|
|
|
print("this node fails all of its health checks - deleting it!")
|
|
|
|
node.get_failed_checks().count() == node.get_all_checks().count()
|
|
|
|
for _hc in node.get_all_checks():
|
|
|
|
and node.get_all_checks().count() > 5
|
|
|
|
_hc.delete_instance()
|
|
|
|
):
|
|
|
|
node.delete_instance()
|
|
|
|
print("this node fails all of its health checks - deleting it!")
|
|
|
|
|
|
|
|
for _hc in node.get_all_checks():
|
|
|
|
|
|
|
|
_hc.delete_instance()
|
|
|
|
|
|
|
|
node.delete_instance()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@bp.cli.command("get_peers")
|
|
|
|
@bp.cli.command("get_peers")
|
|
|
|