fix forms and refactor node ingest/validation

pull/1/head
lza_menace 4 years ago
parent dc06c87808
commit 137a95217d

@ -1,7 +1,7 @@
#!/bin/bash
source .venv/bin/activate
export FLASK_APP=nodes/app.py
export FLASK_APP=xmrnodes/app.py
export FLASK_SECRETS=config.py
export FLASK_DEBUG=1
flask $1

@ -1,19 +1,28 @@
import json
import requests
import re
import logging
from os import makedirs
from datetime import datetime
from flask import Flask, request, redirect
from flask import render_template, flash, url_for
from urllib.parse import urlparse
from xmrnodes.forms import SubmitNode
from xmrnodes.models import Node
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
app = Flask(__name__)
app.config.from_envvar("FLASK_SECRETS")
app.secret_key = app.config["SECRET_KEY"]
@app.route("/", methods=["GET", "POST"])
def index():
form = SubmitNode()
itp = 20
page = request.args.get("page", 1)
try:
@ -22,61 +31,85 @@ def index():
flash("Wow, wtf hackerman. Cool it.")
page = 1
nodes = Node.select().where(Node.available==True).order_by(Node.datetime_entered.desc()).paginate(page, itp)
nodes = Node.select().where(Node.available==True).order_by(
Node.datetime_entered.desc()
).paginate(page, itp)
total_pages = Node.select().count() / itp
return render_template("index.html", nodes=nodes, page=page, total_pages=total_pages)
return render_template(
"index.html",
nodes=nodes,
page=page,
total_pages=total_pages,
form=form
)
@app.route("/add", methods=["GET", "POST"])
def add():
if request.method == "POST":
url = request.form.get("url")
url = request.form.get("node_url")
regex = re.compile(
r'^(?:http)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
r'localhost|' #localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
r'^(?:http)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
r'localhost|' #localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE
)
re_match = re.match(regex, url)
if re_match is not None:
_url = urlparse(url)
try:
endpoint = f"{_url.scheme}://{_url.netloc}"
r = requests.get(endpoint + "/get_info", timeout=3)
r.raise_for_status()
# print(r.json())
return {"status": "success"}
except requests.exceptions.ConnectTimeout:
flash("connection timed out. double-check the port")
return {"status": "fail", "reason": "timeout"}
except requests.exceptions.SSLError:
flash("invalid certificate")
return {"status": "fail", "reason": "invalid cert"}
except Exception as e:
flash("failed to send req", str(e))
print(e)
return {"status": "fail"}
if re_match is None:
flash("This doesn't look like a valid URL")
else:
flash("invalid url provided")
return {"status": "fail"}
return "ok"
node = Node(
scheme=proto,
address=addr,
port=port,
version=r.json()["version"],
tor=addr.endswith(".onion"),
available=r.json()["status"] == "OK",
mainnet=r.json()["mainnet"],
)
node.save()
return {"status": "success"}
_url = urlparse(url)
url = f"{_url.scheme}://{_url.netloc}"
if Node.select().where(Node.url == url).exists():
flash("This node is already in the database.")
else:
flash("Seems like a valid node. Added to the database and will check soon.")
node = Node(url=url)
node.save()
return redirect("/")
@app.cli.command("validate")
def validate():
nodes = Node.select().where(Node.validated == False)
for node in nodes:
now = datetime.now()
is_onion = node.url.split(":")[1].endswith(".onion")
logging.info(f"Attempting to validate {node.url}")
if is_onion:
logging.info("onion address found")
node.tor = True
try:
r = requests.get(node.url + "/get_info", timeout=3)
r.raise_for_status()
assert "height" in r.json()
assert "nettype" in r.json()
nettype = r.json()["nettype"]
logging.info("success")
if nettype in ["mainnet", "stagenet", "testnet"]:
node.nettype = nettype
node.available = True
node.validated = True
node.datetime_checked = now
node.save()
else:
logging.info("unexpected nettype")
except requests.exceptions.ConnectTimeout:
logging.info("connection timed out")
node.delete_instance()
except requests.exceptions.SSLError:
logging.info("invalid certificate")
node.delete_instance()
except requests.exceptions.ConnectionError:
logging.info("connection error")
node.delete_instance()
except requests.exceptions.HTTPError:
logging.info("http error, 4xx or 5xx")
node.delete_instance()
except Exception as e:
logging.info("failed for reasons unknown")
node.delete_instance()
@app.route("/about")
def about():

@ -0,0 +1,7 @@
from flask_wtf import FlaskForm
from wtforms import StringField
from wtforms.validators import DataRequired
class SubmitNode(FlaskForm):
node_url = StringField('Node URL:', validators=[DataRequired()])

@ -4,19 +4,17 @@ from xmrnodes import config
data_dir = getattr(config, 'DATA_FOLDER', './data')
db = SqliteDatabase(f"{data_dir}/db/sqlite.db")
db = SqliteDatabase(f"{data_dir}/sqlite.db")
class Node(Model):
id = AutoField()
scheme = CharField()
address = CharField()
port = IntegerField()
version = CharField(null=True)
url = CharField()
tor = BooleanField(default=False)
available = BooleanField(default=False)
mainnet = BooleanField(default=False)
validated = BooleanField(default=False)
nettype = CharField(null=True)
datetime_entered = DateTimeField(default=datetime.now)
datetime_checked = DateTimeField(default=datetime.now)
datetime_checked = DateTimeField(default=None, null=True)
datetime_failed = DateTimeField(default=None, null=True)
class Meta:

@ -9,13 +9,25 @@
</div>
{% for node in nodes %}
{{ node }}<br>
{{ node.url }}<br>
{% endfor %}
<form id="addnode" method="POST">
<label>Node URL:</label>
<input type="text" name="url"/>
<input type="submit" value="Submit" name="submit" class="submit" id="submit" />
<form method="POST" action="{{ url_for('add') }}">
{{ form.csrf_token }}
{% for f in form %}
{% if f.name != 'csrf_token' %}
<div class="form-group">
{{ f.label }}
{{ f }}
</div>
{% endif %}
{% endfor %}
<ul>
{% for field, errors in form.errors.items() %}
<li>{{ form[field].label }}: {{ ', '.join(errors) }}</li>
{% endfor %}
</ul>
<input type="submit" value="Send" class="btn btn-link btn-outline btn-xl">
</form>

Loading…
Cancel
Save