split resources out
parent
726d2766e8
commit
2fb28e90fb
@ -1,421 +1,23 @@
|
|||||||
import re
|
|
||||||
import logging
|
import logging
|
||||||
from random import shuffle
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
|
|
||||||
import geoip2.database
|
from flask import Flask
|
||||||
import arrow
|
|
||||||
import requests
|
|
||||||
from flask import Flask, request, redirect, jsonify
|
|
||||||
from flask import render_template, flash, Response
|
|
||||||
from urllib.parse import urlparse, urlencode
|
|
||||||
|
|
||||||
from xmrnodes.helpers import determine_crypto, is_onion, make_request
|
from xmrnodes.routes import meta, api
|
||||||
from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
|
from xmrnodes import cli, filters
|
||||||
from xmrnodes.forms import SubmitNode
|
|
||||||
from xmrnodes.models import Node, HealthCheck, Peer
|
|
||||||
from xmrnodes import config
|
|
||||||
|
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||||
format="%(asctime)s - %(levelname)s - %(message)s"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
app.config.from_envvar("FLASK_SECRETS")
|
app.config.from_envvar("FLASK_SECRETS")
|
||||||
app.secret_key = app.config["SECRET_KEY"]
|
app.secret_key = app.config["SECRET_KEY"]
|
||||||
HEALTHY_BLOCK_DIFF = 500 # idc to config this. hardcode is fine.
|
app.register_blueprint(meta.bp)
|
||||||
|
app.register_blueprint(api.bp)
|
||||||
|
app.register_blueprint(cli.bp)
|
||||||
|
app.register_blueprint(filters.bp)
|
||||||
|
|
||||||
@app.route("/", methods=["GET", "POST"])
|
|
||||||
def index():
|
|
||||||
form = SubmitNode()
|
|
||||||
nettype = request.args.get("network", "mainnet")
|
|
||||||
crypto = request.args.get("chain", "monero")
|
|
||||||
onion = request.args.get("onion", False)
|
|
||||||
show_all = "true" == request.args.get("all", "false")
|
|
||||||
web_compatible = request.args.get("cors", False)
|
|
||||||
highest_block = get_highest_block(nettype, crypto)
|
|
||||||
healthy_block = highest_block - HEALTHY_BLOCK_DIFF
|
|
||||||
|
|
||||||
nodes = Node.select().where(
|
|
||||||
Node.validated == True,
|
|
||||||
Node.nettype == nettype,
|
|
||||||
Node.crypto == crypto
|
|
||||||
)
|
|
||||||
|
|
||||||
if web_compatible:
|
|
||||||
nodes = nodes.where(Node.web_compatible == True)
|
|
||||||
|
|
||||||
nodes_all = nodes.count()
|
|
||||||
nodes_unhealthy = nodes.where(
|
|
||||||
(Node.available == False) | (Node.last_height < healthy_block)
|
|
||||||
).count()
|
|
||||||
|
|
||||||
if not show_all:
|
|
||||||
nodes = nodes.where(
|
|
||||||
Node.available == True,
|
|
||||||
Node.last_height > healthy_block
|
|
||||||
)
|
|
||||||
|
|
||||||
nodes = nodes.order_by(
|
|
||||||
Node.datetime_entered.desc()
|
|
||||||
)
|
|
||||||
if onion:
|
|
||||||
nodes = nodes.where(Node.is_tor == True)
|
|
||||||
|
|
||||||
nodes = [n for n in nodes]
|
|
||||||
shuffle(nodes)
|
|
||||||
|
|
||||||
return render_template(
|
|
||||||
"index.html",
|
|
||||||
nodes=nodes,
|
|
||||||
nodes_all=nodes_all,
|
|
||||||
nodes_unhealthy=nodes_unhealthy,
|
|
||||||
nettype=nettype,
|
|
||||||
crypto=crypto,
|
|
||||||
form=form,
|
|
||||||
web_compatible=web_compatible
|
|
||||||
)
|
|
||||||
|
|
||||||
@app.route("/nodes.json")
|
|
||||||
def nodes_json():
|
|
||||||
nodes = Node.select().where(
|
|
||||||
Node.validated==True
|
|
||||||
).where(
|
|
||||||
Node.nettype=="mainnet"
|
|
||||||
)
|
|
||||||
xmr_nodes = [n for n in nodes if n.crypto == "monero"]
|
|
||||||
wow_nodes = [n for n in nodes if n.crypto == "wownero"]
|
|
||||||
return jsonify({
|
|
||||||
"monero": {
|
|
||||||
"clear": [n.url for n in xmr_nodes if n.is_tor == False],
|
|
||||||
"onion": [n.url for n in xmr_nodes if n.is_tor == True],
|
|
||||||
"web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
|
|
||||||
},
|
|
||||||
"wownero": {
|
|
||||||
"clear": [n.url for n in wow_nodes if n.is_tor == False],
|
|
||||||
"onion": [n.url for n in wow_nodes if n.is_tor == True]
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
@app.route("/health.json")
|
|
||||||
def health_json():
|
|
||||||
data = {}
|
|
||||||
nodes = Node.select().where(
|
|
||||||
Node.validated == True
|
|
||||||
)
|
|
||||||
for node in nodes:
|
|
||||||
if node.crypto not in data:
|
|
||||||
data[node.crypto] = {}
|
|
||||||
_d = {
|
|
||||||
"available": node.available,
|
|
||||||
"last_height": node.last_height,
|
|
||||||
"datetime_entered": node.datetime_entered,
|
|
||||||
"datetime_checked": node.datetime_checked,
|
|
||||||
"datetime_failed": node.datetime_failed,
|
|
||||||
"checks": [c.health for c in node.get_all_checks()]
|
|
||||||
}
|
|
||||||
nettype = "clear"
|
|
||||||
if node.is_tor:
|
|
||||||
nettype = "onion"
|
|
||||||
elif node.web_compatible:
|
|
||||||
if "web_compatible" not in data[node.crypto]:
|
|
||||||
data[node.crypto]["web_compatible"] = {}
|
|
||||||
data[node.crypto]["web_compatible"][node.url] = _d
|
|
||||||
if nettype not in data[node.crypto]:
|
|
||||||
data[node.crypto][nettype] = {}
|
|
||||||
data[node.crypto][nettype][node.url] = _d
|
|
||||||
return jsonify(data)
|
|
||||||
|
|
||||||
@app.route("/haproxy.cfg")
|
|
||||||
def haproxy():
|
|
||||||
crypto = request.args.get('chain') or 'monero'
|
|
||||||
nettype = request.args.get('network') or 'mainnet'
|
|
||||||
cors = request.args.get('cors') or False
|
|
||||||
tor = request.args.get('onion') or False
|
|
||||||
nodes = Node.select().where(
|
|
||||||
Node.validated == True,
|
|
||||||
Node.nettype == nettype,
|
|
||||||
Node.crypto == crypto,
|
|
||||||
Node.is_tor == tor,
|
|
||||||
Node.web_compatible == cors
|
|
||||||
)
|
|
||||||
tpl = render_template("haproxy.html", nodes=nodes)
|
|
||||||
print(tpl)
|
|
||||||
res = Response(tpl)
|
|
||||||
res.headers['Content-Disposition'] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
|
|
||||||
return res
|
|
||||||
|
|
||||||
@app.route("/wow_nodes.json")
|
|
||||||
def wow_nodes_json():
|
|
||||||
nodes = Node.select().where(
|
|
||||||
Node.validated==True
|
|
||||||
).where(
|
|
||||||
Node.nettype=="mainnet"
|
|
||||||
).where(
|
|
||||||
Node.crypto=="wownero"
|
|
||||||
)
|
|
||||||
nodes = [n for n in nodes]
|
|
||||||
return jsonify({
|
|
||||||
"clear": [n.url for n in nodes if n.is_tor == False],
|
|
||||||
"onion": [n.url for n in nodes if n.is_tor == True]
|
|
||||||
})
|
|
||||||
|
|
||||||
@app.route("/map")
|
|
||||||
def map():
|
|
||||||
try:
|
|
||||||
peers = rw_cache('map_peers')
|
|
||||||
except:
|
|
||||||
flash('Couldn\'t load the map. Try again later.')
|
|
||||||
return redirect('/')
|
|
||||||
return render_template(
|
|
||||||
"map.html",
|
|
||||||
peers=peers,
|
|
||||||
source_node=config.NODE_HOST
|
|
||||||
)
|
|
||||||
|
|
||||||
@app.route("/resources")
|
|
||||||
def resources():
|
|
||||||
return render_template("resources.html")
|
|
||||||
|
|
||||||
@app.route("/add", methods=["GET", "POST"])
|
|
||||||
def add():
|
|
||||||
if request.method == "POST":
|
|
||||||
url = request.form.get("node_url")
|
|
||||||
regex = re.compile(
|
|
||||||
r"^(?:http)s?://" # http:// or https://
|
|
||||||
r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" #domain...
|
|
||||||
r"localhost|" #localhost...
|
|
||||||
r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip
|
|
||||||
r"(?::\d+)?" # optional port
|
|
||||||
r"(?:/?|[/?]\S+)$", re.IGNORECASE
|
|
||||||
)
|
|
||||||
re_match = re.match(regex, url)
|
|
||||||
if re_match is None:
|
|
||||||
flash("This doesn\"t look like a valid URL")
|
|
||||||
else:
|
|
||||||
_url = urlparse(url)
|
|
||||||
url = f"{_url.scheme}://{_url.netloc}".lower()
|
|
||||||
if Node.select().where(Node.url == url).exists():
|
|
||||||
flash("This node is already in the database.")
|
|
||||||
else:
|
|
||||||
flash("Seems like a valid node URL. Added to the database and will check soon.")
|
|
||||||
node = Node(url=url)
|
|
||||||
node.save()
|
|
||||||
return redirect("/")
|
|
||||||
|
|
||||||
def cleanup_health_checks():
|
|
||||||
diff = datetime.utcnow() - timedelta(hours=24)
|
|
||||||
checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
|
|
||||||
for check in checks:
|
|
||||||
print("Deleting check", check.id)
|
|
||||||
check.delete_instance()
|
|
||||||
|
|
||||||
@app.cli.command("check")
|
|
||||||
def check():
|
|
||||||
cleanup_health_checks()
|
|
||||||
nodes = Node.select().where(Node.validated == True)
|
|
||||||
for node in nodes:
|
|
||||||
now = datetime.utcnow()
|
|
||||||
hc = HealthCheck(node=node)
|
|
||||||
logging.info(f"Attempting to check {node.url}")
|
|
||||||
try:
|
|
||||||
r = make_request(node.url)
|
|
||||||
assert "status" in r.json()
|
|
||||||
assert "offline" in r.json()
|
|
||||||
assert "height" in r.json()
|
|
||||||
has_cors = 'Access-Control-Allow-Origin' in r.headers
|
|
||||||
is_ssl = node.url.startswith('https://')
|
|
||||||
if r.json()["status"] == "OK":
|
|
||||||
node.web_compatible = has_cors and is_ssl
|
|
||||||
node.last_height = r.json()["height"]
|
|
||||||
hc.health = True
|
|
||||||
highest_block = get_highest_block(node.nettype, node.crypto)
|
|
||||||
healthy_block = highest_block - HEALTHY_BLOCK_DIFF
|
|
||||||
if r.json()["height"] < healthy_block:
|
|
||||||
node.available = False
|
|
||||||
logging.info("unhealthy")
|
|
||||||
else:
|
|
||||||
node.available = True
|
|
||||||
logging.info("success")
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
except:
|
|
||||||
logging.info("fail")
|
|
||||||
node.datetime_failed = now
|
|
||||||
node.available = False
|
|
||||||
hc.health = False
|
|
||||||
finally:
|
|
||||||
node.datetime_checked = now
|
|
||||||
node.save()
|
|
||||||
hc.save()
|
|
||||||
if node.get_failed_checks().count() == node.get_all_checks().count() and node.get_all_checks().count() > 5:
|
|
||||||
print('this node fails all of its health checks - deleting it!')
|
|
||||||
for _hc in node.get_all_checks():
|
|
||||||
_hc.delete_instance()
|
|
||||||
node.delete_instance()
|
|
||||||
|
|
||||||
@app.cli.command("get_peers")
|
|
||||||
def get_peers():
|
|
||||||
all_peers = []
|
|
||||||
print('[+] Preparing to crawl Monero p2p network')
|
|
||||||
print(f'[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
|
|
||||||
initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
|
|
||||||
with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
|
|
||||||
for peer in initial_peers:
|
|
||||||
if peer not in all_peers:
|
|
||||||
all_peers.append(peer)
|
|
||||||
_url = urlparse(peer)
|
|
||||||
url = f"{_url.scheme}://{_url.netloc}".lower()
|
|
||||||
if not Peer.select().where(Peer.url == peer).exists():
|
|
||||||
response = reader.city(_url.hostname)
|
|
||||||
p = Peer(
|
|
||||||
url=peer,
|
|
||||||
country=response.country.name,
|
|
||||||
city=response.city.name,
|
|
||||||
postal=response.postal.code,
|
|
||||||
lat=response.location.latitude,
|
|
||||||
lon=response.location.longitude,
|
|
||||||
)
|
|
||||||
p.save()
|
|
||||||
print(f'{peer} - saving new peer')
|
|
||||||
else:
|
|
||||||
p = Peer.select().where(Peer.url == peer).first()
|
|
||||||
p.datetime = datetime.now()
|
|
||||||
p.save()
|
|
||||||
|
|
||||||
try:
|
|
||||||
print(f'[.] Retrieving crawled peers from {_url.netloc}')
|
|
||||||
new_peers = retrieve_peers(_url.hostname, _url.port)
|
|
||||||
for peer in new_peers:
|
|
||||||
if peer not in all_peers:
|
|
||||||
all_peers.append(peer)
|
|
||||||
_url = urlparse(peer)
|
|
||||||
url = f"{_url.scheme}://{_url.netloc}".lower()
|
|
||||||
if not Peer.select().where(Peer.url == peer).exists():
|
|
||||||
response = reader.city(_url.hostname)
|
|
||||||
p = Peer(
|
|
||||||
url=peer,
|
|
||||||
country=response.country.name,
|
|
||||||
city=response.city.name,
|
|
||||||
postal=response.postal.code,
|
|
||||||
lat=response.location.latitude,
|
|
||||||
lon=response.location.longitude,
|
|
||||||
)
|
|
||||||
p.save()
|
|
||||||
print(f'{peer} - saving new peer')
|
|
||||||
else:
|
|
||||||
p = Peer.select().where(Peer.url == peer).first()
|
|
||||||
p.datetime = datetime.now()
|
|
||||||
p.save()
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
print(f'[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}')
|
|
||||||
print('[+] Deleting old Monero p2p peers')
|
|
||||||
for p in Peer.select():
|
|
||||||
if p.hours_elapsed() > 24:
|
|
||||||
print(f'[.] Deleting {p.url}')
|
|
||||||
p.delete_instance()
|
|
||||||
rw_cache('map_peers', list(Peer.select().execute()))
|
|
||||||
|
|
||||||
|
|
||||||
@app.cli.command('init')
|
|
||||||
def init():
|
|
||||||
pass
|
|
||||||
|
|
||||||
@app.cli.command("validate")
|
|
||||||
def validate():
|
|
||||||
nodes = Node.select().where(Node.validated == False)
|
|
||||||
for node in nodes:
|
|
||||||
now = datetime.utcnow()
|
|
||||||
logging.info(f"Attempting to validate {node.url}")
|
|
||||||
try:
|
|
||||||
r = make_request(node.url)
|
|
||||||
assert "height" in r.json()
|
|
||||||
assert "nettype" in r.json()
|
|
||||||
has_cors = 'Access-Control-Allow-Origin' in r.headers
|
|
||||||
is_ssl = node.url.startswith('https://')
|
|
||||||
nettype = r.json()["nettype"]
|
|
||||||
crypto = determine_crypto(node.url)
|
|
||||||
logging.info("success")
|
|
||||||
if nettype in ["mainnet", "stagenet", "testnet"]:
|
|
||||||
node.nettype = nettype
|
|
||||||
node.available = True
|
|
||||||
node.validated = True
|
|
||||||
node.web_compatible = has_cors and is_ssl
|
|
||||||
node.last_height = r.json()["height"]
|
|
||||||
node.datetime_checked = now
|
|
||||||
node.crypto = crypto
|
|
||||||
node.is_tor = is_onion(node.url)
|
|
||||||
node.save()
|
|
||||||
else:
|
|
||||||
logging.info("unexpected nettype")
|
|
||||||
except requests.exceptions.ConnectTimeout:
|
|
||||||
logging.info("connection timed out")
|
|
||||||
node.delete_instance()
|
|
||||||
except requests.exceptions.SSLError:
|
|
||||||
logging.info("invalid certificate")
|
|
||||||
node.delete_instance()
|
|
||||||
except requests.exceptions.ConnectionError:
|
|
||||||
logging.info("connection error")
|
|
||||||
node.delete_instance()
|
|
||||||
except requests.exceptions.HTTPError:
|
|
||||||
logging.info("http error, 4xx or 5xx")
|
|
||||||
node.delete_instance()
|
|
||||||
except Exception as e:
|
|
||||||
logging.info("failed for reasons unknown")
|
|
||||||
node.delete_instance()
|
|
||||||
|
|
||||||
@app.cli.command("export")
|
|
||||||
def export():
|
|
||||||
all_nodes = []
|
|
||||||
ts = int(arrow.get().timestamp())
|
|
||||||
export_dir = f"{config.DATA_DIR}/export.txt"
|
|
||||||
export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
|
|
||||||
nodes = Node.select().where(Node.validated == True)
|
|
||||||
for node in nodes:
|
|
||||||
logging.info(f"Adding {node.url}")
|
|
||||||
all_nodes.append(node.url)
|
|
||||||
with open(export_dir, "w") as f:
|
|
||||||
f.write("\n".join(all_nodes))
|
|
||||||
with open(export_dir_stamped, "w") as f:
|
|
||||||
f.write("\n".join(all_nodes))
|
|
||||||
logging.info(f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}")
|
|
||||||
|
|
||||||
@app.cli.command("import")
|
|
||||||
def import_():
|
|
||||||
all_nodes = []
|
|
||||||
export_dir = f"{config.DATA_DIR}/export.txt"
|
|
||||||
with open(export_dir, "r") as f:
|
|
||||||
for url in f.readlines():
|
|
||||||
try:
|
|
||||||
n = url.rstrip().lower()
|
|
||||||
logging.info(f"Adding {n}")
|
|
||||||
node = Node(url=n)
|
|
||||||
node.save()
|
|
||||||
all_nodes.append(n)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")
|
|
||||||
|
|
||||||
@app.template_filter("humanize")
|
|
||||||
def humanize(d):
|
|
||||||
t = arrow.get(d, "UTC")
|
|
||||||
return t.humanize()
|
|
||||||
|
|
||||||
@app.template_filter("hours_elapsed")
|
|
||||||
def hours_elapsed(d):
|
|
||||||
now = datetime.utcnow()
|
|
||||||
diff = now - d
|
|
||||||
return diff.total_seconds() / 60 / 60
|
|
||||||
|
|
||||||
@app.template_filter("pop_arg")
|
|
||||||
def trim_arg(all_args, arg_to_trim):
|
|
||||||
d = all_args.to_dict()
|
|
||||||
d.pop(arg_to_trim)
|
|
||||||
return urlencode(d)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app.run()
|
app.run()
|
||||||
|
|||||||
@ -0,0 +1,219 @@
|
|||||||
|
import logging
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import geoip2.database
|
||||||
|
import arrow
|
||||||
|
import requests
|
||||||
|
from flask import Blueprint
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from xmrnodes.helpers import determine_crypto, is_onion, make_request
|
||||||
|
from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
|
||||||
|
from xmrnodes.models import Node, HealthCheck, Peer
|
||||||
|
from xmrnodes import config
|
||||||
|
|
||||||
|
bp = Blueprint("cli", "cli", cli_group=None)
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("init")
|
||||||
|
def init():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("check")
|
||||||
|
def check():
|
||||||
|
diff = datetime.utcnow() - timedelta(hours=24)
|
||||||
|
checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
|
||||||
|
for check in checks:
|
||||||
|
print("Deleting check", check.id)
|
||||||
|
check.delete_instance()
|
||||||
|
nodes = Node.select().where(Node.validated == True)
|
||||||
|
for node in nodes:
|
||||||
|
now = datetime.utcnow()
|
||||||
|
hc = HealthCheck(node=node)
|
||||||
|
logging.info(f"Attempting to check {node.url}")
|
||||||
|
try:
|
||||||
|
r = make_request(node.url)
|
||||||
|
assert "status" in r.json()
|
||||||
|
assert "offline" in r.json()
|
||||||
|
assert "height" in r.json()
|
||||||
|
has_cors = "Access-Control-Allow-Origin" in r.headers
|
||||||
|
is_ssl = node.url.startswith("https://")
|
||||||
|
if r.json()["status"] == "OK":
|
||||||
|
node.web_compatible = has_cors and is_ssl
|
||||||
|
node.last_height = r.json()["height"]
|
||||||
|
hc.health = True
|
||||||
|
highest_block = get_highest_block(node.nettype, node.crypto)
|
||||||
|
healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
|
||||||
|
if r.json()["height"] < healthy_block:
|
||||||
|
node.available = False
|
||||||
|
logging.info("unhealthy")
|
||||||
|
else:
|
||||||
|
node.available = True
|
||||||
|
logging.info("success")
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
except:
|
||||||
|
logging.info("fail")
|
||||||
|
node.datetime_failed = now
|
||||||
|
node.available = False
|
||||||
|
hc.health = False
|
||||||
|
finally:
|
||||||
|
node.datetime_checked = now
|
||||||
|
node.save()
|
||||||
|
hc.save()
|
||||||
|
if (
|
||||||
|
node.get_failed_checks().count() == node.get_all_checks().count()
|
||||||
|
and node.get_all_checks().count() > 5
|
||||||
|
):
|
||||||
|
print("this node fails all of its health checks - deleting it!")
|
||||||
|
for _hc in node.get_all_checks():
|
||||||
|
_hc.delete_instance()
|
||||||
|
node.delete_instance()
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("get_peers")
|
||||||
|
def get_peers():
|
||||||
|
all_peers = []
|
||||||
|
print("[+] Preparing to crawl Monero p2p network")
|
||||||
|
print(f"[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}")
|
||||||
|
initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
|
||||||
|
with geoip2.database.Reader("./data/GeoLite2-City.mmdb") as reader:
|
||||||
|
for peer in initial_peers:
|
||||||
|
if peer not in all_peers:
|
||||||
|
all_peers.append(peer)
|
||||||
|
_url = urlparse(peer)
|
||||||
|
url = f"{_url.scheme}://{_url.netloc}".lower()
|
||||||
|
if not Peer.select().where(Peer.url == peer).exists():
|
||||||
|
response = reader.city(_url.hostname)
|
||||||
|
p = Peer(
|
||||||
|
url=peer,
|
||||||
|
country=response.country.name,
|
||||||
|
city=response.city.name,
|
||||||
|
postal=response.postal.code,
|
||||||
|
lat=response.location.latitude,
|
||||||
|
lon=response.location.longitude,
|
||||||
|
)
|
||||||
|
p.save()
|
||||||
|
print(f"{peer} - saving new peer")
|
||||||
|
else:
|
||||||
|
p = Peer.select().where(Peer.url == peer).first()
|
||||||
|
p.datetime = datetime.now()
|
||||||
|
p.save()
|
||||||
|
|
||||||
|
try:
|
||||||
|
print(f"[.] Retrieving crawled peers from {_url.netloc}")
|
||||||
|
new_peers = retrieve_peers(_url.hostname, _url.port)
|
||||||
|
for peer in new_peers:
|
||||||
|
if peer not in all_peers:
|
||||||
|
all_peers.append(peer)
|
||||||
|
_url = urlparse(peer)
|
||||||
|
url = f"{_url.scheme}://{_url.netloc}".lower()
|
||||||
|
if not Peer.select().where(Peer.url == peer).exists():
|
||||||
|
response = reader.city(_url.hostname)
|
||||||
|
p = Peer(
|
||||||
|
url=peer,
|
||||||
|
country=response.country.name,
|
||||||
|
city=response.city.name,
|
||||||
|
postal=response.postal.code,
|
||||||
|
lat=response.location.latitude,
|
||||||
|
lon=response.location.longitude,
|
||||||
|
)
|
||||||
|
p.save()
|
||||||
|
print(f"{peer} - saving new peer")
|
||||||
|
else:
|
||||||
|
p = Peer.select().where(Peer.url == peer).first()
|
||||||
|
p.datetime = datetime.now()
|
||||||
|
p.save()
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}"
|
||||||
|
)
|
||||||
|
print("[+] Deleting old Monero p2p peers")
|
||||||
|
for p in Peer.select():
|
||||||
|
if p.hours_elapsed() > 24:
|
||||||
|
print(f"[.] Deleting {p.url}")
|
||||||
|
p.delete_instance()
|
||||||
|
rw_cache("map_peers", list(Peer.select().execute()))
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("validate")
|
||||||
|
def validate():
|
||||||
|
nodes = Node.select().where(Node.validated == False)
|
||||||
|
for node in nodes:
|
||||||
|
now = datetime.utcnow()
|
||||||
|
logging.info(f"Attempting to validate {node.url}")
|
||||||
|
try:
|
||||||
|
r = make_request(node.url)
|
||||||
|
assert "height" in r.json()
|
||||||
|
assert "nettype" in r.json()
|
||||||
|
has_cors = "Access-Control-Allow-Origin" in r.headers
|
||||||
|
is_ssl = node.url.startswith("https://")
|
||||||
|
nettype = r.json()["nettype"]
|
||||||
|
crypto = determine_crypto(node.url)
|
||||||
|
logging.info("success")
|
||||||
|
if nettype in ["mainnet", "stagenet", "testnet"]:
|
||||||
|
node.nettype = nettype
|
||||||
|
node.available = True
|
||||||
|
node.validated = True
|
||||||
|
node.web_compatible = has_cors and is_ssl
|
||||||
|
node.last_height = r.json()["height"]
|
||||||
|
node.datetime_checked = now
|
||||||
|
node.crypto = crypto
|
||||||
|
node.is_tor = is_onion(node.url)
|
||||||
|
node.save()
|
||||||
|
else:
|
||||||
|
logging.info("unexpected nettype")
|
||||||
|
except requests.exceptions.ConnectTimeout:
|
||||||
|
logging.info("connection timed out")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.SSLError:
|
||||||
|
logging.info("invalid certificate")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
logging.info("connection error")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.HTTPError:
|
||||||
|
logging.info("http error, 4xx or 5xx")
|
||||||
|
node.delete_instance()
|
||||||
|
except Exception as e:
|
||||||
|
logging.info("failed for reasons unknown")
|
||||||
|
node.delete_instance()
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("export")
|
||||||
|
def export():
|
||||||
|
all_nodes = []
|
||||||
|
ts = int(arrow.get().timestamp())
|
||||||
|
export_dir = f"{config.DATA_DIR}/export.txt"
|
||||||
|
export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
|
||||||
|
nodes = Node.select().where(Node.validated == True)
|
||||||
|
for node in nodes:
|
||||||
|
logging.info(f"Adding {node.url}")
|
||||||
|
all_nodes.append(node.url)
|
||||||
|
with open(export_dir, "w") as f:
|
||||||
|
f.write("\n".join(all_nodes))
|
||||||
|
with open(export_dir_stamped, "w") as f:
|
||||||
|
f.write("\n".join(all_nodes))
|
||||||
|
logging.info(
|
||||||
|
f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@bp.cli.command("import")
|
||||||
|
def import_():
|
||||||
|
all_nodes = []
|
||||||
|
export_dir = f"{config.DATA_DIR}/export.txt"
|
||||||
|
with open(export_dir, "r") as f:
|
||||||
|
for url in f.readlines():
|
||||||
|
try:
|
||||||
|
n = url.rstrip().lower()
|
||||||
|
logging.info(f"Adding {n}")
|
||||||
|
node = Node(url=n)
|
||||||
|
node.save()
|
||||||
|
all_nodes.append(n)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")
|
||||||
@ -1,8 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
SECRET_KEY = os.environ.get('SECRET_KEY', 'xxxx')
|
|
||||||
DATA_DIR = os.environ.get('DATA_DIR', './data')
|
|
||||||
TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
|
|
||||||
TOR_PORT = os.environ.get('TOR_PORT', 9050)
|
|
||||||
NODE_HOST = os.environ.get('NODE_HOST', '127.0.0.1')
|
|
||||||
NODE_PORT = os.environ.get('NODE_PORT', 18080)
|
|
||||||
@ -1,14 +1,15 @@
|
|||||||
import os
|
from os import environ
|
||||||
from secrets import token_urlsafe
|
from secrets import token_urlsafe
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
SECRET_KEY = os.environ.get('SECRET_KEY', token_urlsafe(14))
|
SECRET_KEY = environ.get("SECRET_KEY", token_urlsafe(14))
|
||||||
SERVER_NAME = os.environ.get('SERVER_NAME', '127.0.0.1:5000')
|
SERVER_NAME = environ.get("SERVER_NAME", "127.0.0.1:5000")
|
||||||
DATA_DIR = os.environ.get('DATA_DIR', './data')
|
DATA_DIR = environ.get("DATA_DIR", "./data")
|
||||||
TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
|
TOR_HOST = environ.get("TOR_HOST", "127.0.0.1")
|
||||||
TOR_PORT = os.environ.get('TOR_PORT', 9050)
|
TOR_PORT = environ.get("TOR_PORT", 9050)
|
||||||
NODE_HOST = os.environ.get('NODE_HOST', 'singapore.node.xmr.pm')
|
NODE_HOST = environ.get("NODE_HOST", "singapore.node.xmr.pm")
|
||||||
NODE_PORT = os.environ.get('NODE_PORT', 18080)
|
NODE_PORT = environ.get("NODE_PORT", 18080)
|
||||||
|
HEALTHY_BLOCK_DIFF = int(environ.get("HEALTHY_BLOCK_DIFF", 500))
|
||||||
|
|||||||
@ -0,0 +1,27 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
import arrow
|
||||||
|
from flask import Blueprint
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
bp = Blueprint("filters", "filters")
|
||||||
|
|
||||||
|
|
||||||
|
@bp.app_template_filter("humanize")
|
||||||
|
def humanize(d):
|
||||||
|
t = arrow.get(d, "UTC")
|
||||||
|
return t.humanize()
|
||||||
|
|
||||||
|
|
||||||
|
@bp.app_template_filter("hours_elapsed")
|
||||||
|
def hours_elapsed(d):
|
||||||
|
now = datetime.utcnow()
|
||||||
|
diff = now - d
|
||||||
|
return diff.total_seconds() / 60 / 60
|
||||||
|
|
||||||
|
|
||||||
|
@bp.app_template_filter("pop_arg")
|
||||||
|
def trim_arg(all_args, arg_to_trim):
|
||||||
|
d = all_args.to_dict()
|
||||||
|
d.pop(arg_to_trim)
|
||||||
|
return urlencode(d)
|
||||||
@ -0,0 +1,71 @@
|
|||||||
|
from flask import jsonify, Blueprint
|
||||||
|
|
||||||
|
from xmrnodes.models import Node
|
||||||
|
|
||||||
|
|
||||||
|
bp = Blueprint('api', 'api')
|
||||||
|
|
||||||
|
@bp.route("/nodes.json")
|
||||||
|
def nodes_json():
|
||||||
|
nodes = Node.select().where(
|
||||||
|
Node.validated==True
|
||||||
|
).where(
|
||||||
|
Node.nettype=="mainnet"
|
||||||
|
)
|
||||||
|
xmr_nodes = [n for n in nodes if n.crypto == "monero"]
|
||||||
|
wow_nodes = [n for n in nodes if n.crypto == "wownero"]
|
||||||
|
return jsonify({
|
||||||
|
"monero": {
|
||||||
|
"clear": [n.url for n in xmr_nodes if n.is_tor == False],
|
||||||
|
"onion": [n.url for n in xmr_nodes if n.is_tor == True],
|
||||||
|
"web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
|
||||||
|
},
|
||||||
|
"wownero": {
|
||||||
|
"clear": [n.url for n in wow_nodes if n.is_tor == False],
|
||||||
|
"onion": [n.url for n in wow_nodes if n.is_tor == True]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
@bp.route("/health.json")
|
||||||
|
def health_json():
|
||||||
|
data = {}
|
||||||
|
nodes = Node.select().where(
|
||||||
|
Node.validated == True
|
||||||
|
)
|
||||||
|
for node in nodes:
|
||||||
|
if node.crypto not in data:
|
||||||
|
data[node.crypto] = {}
|
||||||
|
_d = {
|
||||||
|
"available": node.available,
|
||||||
|
"last_height": node.last_height,
|
||||||
|
"datetime_entered": node.datetime_entered,
|
||||||
|
"datetime_checked": node.datetime_checked,
|
||||||
|
"datetime_failed": node.datetime_failed,
|
||||||
|
"checks": [c.health for c in node.get_all_checks()]
|
||||||
|
}
|
||||||
|
nettype = "clear"
|
||||||
|
if node.is_tor:
|
||||||
|
nettype = "onion"
|
||||||
|
elif node.web_compatible:
|
||||||
|
if "web_compatible" not in data[node.crypto]:
|
||||||
|
data[node.crypto]["web_compatible"] = {}
|
||||||
|
data[node.crypto]["web_compatible"][node.url] = _d
|
||||||
|
if nettype not in data[node.crypto]:
|
||||||
|
data[node.crypto][nettype] = {}
|
||||||
|
data[node.crypto][nettype][node.url] = _d
|
||||||
|
return jsonify(data)
|
||||||
|
|
||||||
|
@bp.route("/wow_nodes.json")
|
||||||
|
def wow_nodes_json():
|
||||||
|
nodes = Node.select().where(
|
||||||
|
Node.validated==True
|
||||||
|
).where(
|
||||||
|
Node.nettype=="mainnet"
|
||||||
|
).where(
|
||||||
|
Node.crypto=="wownero"
|
||||||
|
)
|
||||||
|
nodes = [n for n in nodes]
|
||||||
|
return jsonify({
|
||||||
|
"clear": [n.url for n in nodes if n.is_tor == False],
|
||||||
|
"onion": [n.url for n in nodes if n.is_tor == True]
|
||||||
|
})
|
||||||
@ -0,0 +1,124 @@
|
|||||||
|
import re
|
||||||
|
from random import shuffle
|
||||||
|
|
||||||
|
from flask import request, redirect, Blueprint
|
||||||
|
from flask import render_template, flash, Response
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from xmrnodes.helpers import rw_cache, get_highest_block
|
||||||
|
from xmrnodes.forms import SubmitNode
|
||||||
|
from xmrnodes.models import Node
|
||||||
|
from xmrnodes import config
|
||||||
|
|
||||||
|
bp = Blueprint("meta", "meta")
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/", methods=["GET", "POST"])
|
||||||
|
def index():
|
||||||
|
form = SubmitNode()
|
||||||
|
nettype = request.args.get("network", "mainnet")
|
||||||
|
crypto = request.args.get("chain", "monero")
|
||||||
|
onion = request.args.get("onion", False)
|
||||||
|
show_all = "true" == request.args.get("all", "false")
|
||||||
|
web_compatible = request.args.get("cors", False)
|
||||||
|
highest_block = get_highest_block(nettype, crypto)
|
||||||
|
healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
|
||||||
|
|
||||||
|
nodes = Node.select().where(
|
||||||
|
Node.validated == True, Node.nettype == nettype, Node.crypto == crypto
|
||||||
|
)
|
||||||
|
|
||||||
|
if web_compatible:
|
||||||
|
nodes = nodes.where(Node.web_compatible == True)
|
||||||
|
|
||||||
|
nodes_all = nodes.count()
|
||||||
|
nodes_unhealthy = nodes.where(
|
||||||
|
(Node.available == False) | (Node.last_height < healthy_block)
|
||||||
|
).count()
|
||||||
|
|
||||||
|
if not show_all:
|
||||||
|
nodes = nodes.where(Node.available == True, Node.last_height > healthy_block)
|
||||||
|
|
||||||
|
nodes = nodes.order_by(Node.datetime_entered.desc())
|
||||||
|
if onion:
|
||||||
|
nodes = nodes.where(Node.is_tor == True)
|
||||||
|
|
||||||
|
nodes = [n for n in nodes]
|
||||||
|
shuffle(nodes)
|
||||||
|
|
||||||
|
return render_template(
|
||||||
|
"index.html",
|
||||||
|
nodes=nodes,
|
||||||
|
nodes_all=nodes_all,
|
||||||
|
nodes_unhealthy=nodes_unhealthy,
|
||||||
|
nettype=nettype,
|
||||||
|
crypto=crypto,
|
||||||
|
form=form,
|
||||||
|
web_compatible=web_compatible,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/map")
|
||||||
|
def map():
|
||||||
|
try:
|
||||||
|
peers = rw_cache("map_peers")
|
||||||
|
except:
|
||||||
|
flash("Couldn't load the map. Try again later.")
|
||||||
|
return redirect("/")
|
||||||
|
return render_template("map.html", peers=peers, source_node=config.NODE_HOST)
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/resources")
|
||||||
|
def resources():
|
||||||
|
return render_template("resources.html")
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/add", methods=["GET", "POST"])
|
||||||
|
def add():
|
||||||
|
if request.method == "POST":
|
||||||
|
url = request.form.get("node_url")
|
||||||
|
regex = re.compile(
|
||||||
|
r"^(?:http)s?://" # http:// or https://
|
||||||
|
r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" # domain...
|
||||||
|
r"localhost|" # localhost...
|
||||||
|
r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip
|
||||||
|
r"(?::\d+)?" # optional port
|
||||||
|
r"(?:/?|[/?]\S+)$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
re_match = re.match(regex, url)
|
||||||
|
if re_match is None:
|
||||||
|
flash("This doesn't look like a valid URL")
|
||||||
|
else:
|
||||||
|
_url = urlparse(url)
|
||||||
|
url = f"{_url.scheme}://{_url.netloc}".lower()
|
||||||
|
if Node.select().where(Node.url == url).exists():
|
||||||
|
flash("This node is already in the database.")
|
||||||
|
else:
|
||||||
|
flash(
|
||||||
|
"Seems like a valid node URL. Added to the database and will check soon."
|
||||||
|
)
|
||||||
|
node = Node(url=url)
|
||||||
|
node.save()
|
||||||
|
return redirect("/")
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/haproxy.cfg")
|
||||||
|
def haproxy():
|
||||||
|
crypto = request.args.get("chain") or "monero"
|
||||||
|
nettype = request.args.get("network") or "mainnet"
|
||||||
|
cors = request.args.get("cors") or False
|
||||||
|
tor = request.args.get("onion") or False
|
||||||
|
nodes = Node.select().where(
|
||||||
|
Node.validated == True,
|
||||||
|
Node.nettype == nettype,
|
||||||
|
Node.crypto == crypto,
|
||||||
|
Node.is_tor == tor,
|
||||||
|
Node.web_compatible == cors,
|
||||||
|
)
|
||||||
|
tpl = render_template("haproxy.html", nodes=nodes)
|
||||||
|
res = Response(tpl)
|
||||||
|
res.headers[
|
||||||
|
"Content-Disposition"
|
||||||
|
] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
|
||||||
|
return res
|
||||||
Loading…
Reference in New Issue