Automatically remove dead nodes (#18)

* remove nodes if all the health checks are failed

* use utc

* node should have at least 5 health checks
backups
lalanza808 2 years ago committed by GitHub
parent b0df1f66c4
commit c78fe4f9f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -157,7 +157,7 @@ def add():
return redirect("/") return redirect("/")
def cleanup_health_checks(): def cleanup_health_checks():
diff = datetime.now() - timedelta(hours=24) diff = datetime.utcnow() - timedelta(hours=24)
checks = HealthCheck.select().where(HealthCheck.datetime <= diff) checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
for check in checks: for check in checks:
print("Deleting check", check.id) print("Deleting check", check.id)
@ -201,6 +201,11 @@ def check():
node.datetime_checked = now node.datetime_checked = now
node.save() node.save()
hc.save() hc.save()
if node.get_failed_checks().count() == node.get_all_checks().count() and node.get_all_checks().count() > 5:
print('this node fails all of its health checks - deleting it!')
for _hc in node.get_all_checks():
_hc.delete_instance()
node.delete_instance()
@app.cli.command("get_peers") @app.cli.command("get_peers")
def get_peers(): def get_peers():

@ -23,6 +23,14 @@ class Node(Model):
datetime_failed = DateTimeField(default=None, null=True) datetime_failed = DateTimeField(default=None, null=True)
fail_reason = CharField(null=True) fail_reason = CharField(null=True)
def get_failed_checks(self):
hcs = HealthCheck.select().where(HealthCheck.node == self, HealthCheck.health == False)
return hcs
def get_all_checks(self):
hcs = HealthCheck.select().where(HealthCheck.node == self)
return hcs
class Meta: class Meta:
database = db database = db

Loading…
Cancel
Save