fix forms and refactor node ingest/validation

pull/1/head
lza_menace 4 years ago
parent dc06c87808
commit 137a95217d

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
source .venv/bin/activate source .venv/bin/activate
export FLASK_APP=nodes/app.py export FLASK_APP=xmrnodes/app.py
export FLASK_SECRETS=config.py export FLASK_SECRETS=config.py
export FLASK_DEBUG=1 export FLASK_DEBUG=1
flask $1 flask $1

@ -1,19 +1,28 @@
import json import json
import requests import requests
import re import re
import logging
from os import makedirs from os import makedirs
from datetime import datetime
from flask import Flask, request, redirect from flask import Flask, request, redirect
from flask import render_template, flash, url_for from flask import render_template, flash, url_for
from urllib.parse import urlparse from urllib.parse import urlparse
from xmrnodes.forms import SubmitNode
from xmrnodes.models import Node from xmrnodes.models import Node
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
app = Flask(__name__) app = Flask(__name__)
app.config.from_envvar("FLASK_SECRETS") app.config.from_envvar("FLASK_SECRETS")
app.secret_key = app.config["SECRET_KEY"] app.secret_key = app.config["SECRET_KEY"]
@app.route("/", methods=["GET", "POST"]) @app.route("/", methods=["GET", "POST"])
def index(): def index():
form = SubmitNode()
itp = 20 itp = 20
page = request.args.get("page", 1) page = request.args.get("page", 1)
try: try:
@ -22,61 +31,85 @@ def index():
flash("Wow, wtf hackerman. Cool it.") flash("Wow, wtf hackerman. Cool it.")
page = 1 page = 1
nodes = Node.select().where(Node.available==True).order_by(Node.datetime_entered.desc()).paginate(page, itp) nodes = Node.select().where(Node.available==True).order_by(
Node.datetime_entered.desc()
).paginate(page, itp)
total_pages = Node.select().count() / itp total_pages = Node.select().count() / itp
return render_template("index.html", nodes=nodes, page=page, total_pages=total_pages) return render_template(
"index.html",
nodes=nodes,
page=page,
total_pages=total_pages,
form=form
)
@app.route("/add", methods=["GET", "POST"]) @app.route("/add", methods=["GET", "POST"])
def add(): def add():
if request.method == "POST": if request.method == "POST":
url = request.form.get("url") url = request.form.get("node_url")
regex = re.compile( regex = re.compile(
r'^(?:http)s?://' # http:// or https:// r'^(?:http)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
r'localhost|' #localhost... r'localhost|' #localhost...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
r'(?::\d+)?' # optional port r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE) r'(?:/?|[/?]\S+)$', re.IGNORECASE
)
re_match = re.match(regex, url) re_match = re.match(regex, url)
if re_match is None:
if re_match is not None: flash("This doesn't look like a valid URL")
_url = urlparse(url)
try:
endpoint = f"{_url.scheme}://{_url.netloc}"
r = requests.get(endpoint + "/get_info", timeout=3)
r.raise_for_status()
# print(r.json())
return {"status": "success"}
except requests.exceptions.ConnectTimeout:
flash("connection timed out. double-check the port")
return {"status": "fail", "reason": "timeout"}
except requests.exceptions.SSLError:
flash("invalid certificate")
return {"status": "fail", "reason": "invalid cert"}
except Exception as e:
flash("failed to send req", str(e))
print(e)
return {"status": "fail"}
else: else:
flash("invalid url provided") _url = urlparse(url)
return {"status": "fail"} url = f"{_url.scheme}://{_url.netloc}"
if Node.select().where(Node.url == url).exists():
return "ok" flash("This node is already in the database.")
node = Node( else:
scheme=proto, flash("Seems like a valid node. Added to the database and will check soon.")
address=addr, node = Node(url=url)
port=port, node.save()
version=r.json()["version"],
tor=addr.endswith(".onion"),
available=r.json()["status"] == "OK",
mainnet=r.json()["mainnet"],
)
node.save()
return {"status": "success"}
return redirect("/") return redirect("/")
@app.cli.command("validate")
def validate():
nodes = Node.select().where(Node.validated == False)
for node in nodes:
now = datetime.now()
is_onion = node.url.split(":")[1].endswith(".onion")
logging.info(f"Attempting to validate {node.url}")
if is_onion:
logging.info("onion address found")
node.tor = True
try:
r = requests.get(node.url + "/get_info", timeout=3)
r.raise_for_status()
assert "height" in r.json()
assert "nettype" in r.json()
nettype = r.json()["nettype"]
logging.info("success")
if nettype in ["mainnet", "stagenet", "testnet"]:
node.nettype = nettype
node.available = True
node.validated = True
node.datetime_checked = now
node.save()
else:
logging.info("unexpected nettype")
except requests.exceptions.ConnectTimeout:
logging.info("connection timed out")
node.delete_instance()
except requests.exceptions.SSLError:
logging.info("invalid certificate")
node.delete_instance()
except requests.exceptions.ConnectionError:
logging.info("connection error")
node.delete_instance()
except requests.exceptions.HTTPError:
logging.info("http error, 4xx or 5xx")
node.delete_instance()
except Exception as e:
logging.info("failed for reasons unknown")
node.delete_instance()
@app.route("/about") @app.route("/about")
def about(): def about():

@ -0,0 +1,7 @@
from flask_wtf import FlaskForm
from wtforms import StringField
from wtforms.validators import DataRequired
class SubmitNode(FlaskForm):
node_url = StringField('Node URL:', validators=[DataRequired()])

@ -4,19 +4,17 @@ from xmrnodes import config
data_dir = getattr(config, 'DATA_FOLDER', './data') data_dir = getattr(config, 'DATA_FOLDER', './data')
db = SqliteDatabase(f"{data_dir}/db/sqlite.db") db = SqliteDatabase(f"{data_dir}/sqlite.db")
class Node(Model): class Node(Model):
id = AutoField() id = AutoField()
scheme = CharField() url = CharField()
address = CharField()
port = IntegerField()
version = CharField(null=True)
tor = BooleanField(default=False) tor = BooleanField(default=False)
available = BooleanField(default=False) available = BooleanField(default=False)
mainnet = BooleanField(default=False) validated = BooleanField(default=False)
nettype = CharField(null=True)
datetime_entered = DateTimeField(default=datetime.now) datetime_entered = DateTimeField(default=datetime.now)
datetime_checked = DateTimeField(default=datetime.now) datetime_checked = DateTimeField(default=None, null=True)
datetime_failed = DateTimeField(default=None, null=True) datetime_failed = DateTimeField(default=None, null=True)
class Meta: class Meta:

@ -9,13 +9,25 @@
</div> </div>
{% for node in nodes %} {% for node in nodes %}
{{ node }}<br> {{ node.url }}<br>
{% endfor %} {% endfor %}
<form id="addnode" method="POST"> <form method="POST" action="{{ url_for('add') }}">
<label>Node URL:</label> {{ form.csrf_token }}
<input type="text" name="url"/> {% for f in form %}
<input type="submit" value="Submit" name="submit" class="submit" id="submit" /> {% if f.name != 'csrf_token' %}
<div class="form-group">
{{ f.label }}
{{ f }}
</div>
{% endif %}
{% endfor %}
<ul>
{% for field, errors in form.errors.items() %}
<li>{{ form[field].label }}: {{ ', '.join(errors) }}</li>
{% endfor %}
</ul>
<input type="submit" value="Send" class="btn btn-link btn-outline btn-xl">
</form> </form>

Loading…
Cancel
Save