diff --git a/scrape.py b/scrape.py
index 53d6487..34ef8ff 100755
--- a/scrape.py
+++ b/scrape.py
@@ -4,28 +4,28 @@ import os
import requests
import bs4
-os.system('mkdir -p infodump/thumbs')
-url = 'https://moneroinfodump.neocities.org/'
+os.system("mkdir -p infodump/thumbs")
+url = "https://moneroinfodump.neocities.org/"
contents = requests.get(url, timeout=15).content
-soup = bs4.BeautifulSoup(contents, 'html.parser')
-images = soup.find_all('img')
-links = soup.find_all('a')
+soup = bs4.BeautifulSoup(contents, "html.parser")
+images = soup.find_all("img")
+links = soup.find_all("a")
for image in images:
- img = image.get('src')
- if img.startswith('http'):
- os.system(f'wget -q --no-clobber -O infodump/{os.path.basename(img)} {img}')
- image['src'] = os.path.basename(img)
- elif img.startswith('data:image/png'):
+ img = image.get("src")
+ if img.startswith("http"):
+ os.system(f"wget -q --no-clobber -O infodump/{os.path.basename(img)} {img}")
+ image["src"] = os.path.basename(img)
+ elif img.startswith("data:image/png"):
pass
else:
- os.system(f'wget -q --no-clobber -O infodump/{img} {img}')
- image['src'] = img
+ os.system(f"wget -q --no-clobber -O infodump/{img} {img}")
+ image["src"] = img
for link in links:
- href = link.get('href')
- if href and href.startswith('https://i.imgur.com'):
- link['href'] = os.path.basename(href)
+ href = link.get("href")
+ if href and href.startswith("https://i.imgur.com"):
+ link["href"] = os.path.basename(href)
-with open('infodump/index.html', 'w') as f:
- f.write(str(soup))
\ No newline at end of file
+with open("infodump/index.html", "w") as f:
+ f.write(str(soup))
diff --git a/xmrnodes/app.py b/xmrnodes/app.py
index c466541..64c0296 100644
--- a/xmrnodes/app.py
+++ b/xmrnodes/app.py
@@ -1,421 +1,23 @@
-import re
import logging
-from random import shuffle
-from datetime import datetime, timedelta
-import geoip2.database
-import arrow
-import requests
-from flask import Flask, request, redirect, jsonify
-from flask import render_template, flash, Response
-from urllib.parse import urlparse, urlencode
+from flask import Flask
-from xmrnodes.helpers import determine_crypto, is_onion, make_request
-from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
-from xmrnodes.forms import SubmitNode
-from xmrnodes.models import Node, HealthCheck, Peer
-from xmrnodes import config
+from xmrnodes.routes import meta, api
+from xmrnodes import cli, filters
logging.basicConfig(
- level=logging.INFO,
- format="%(asctime)s - %(levelname)s - %(message)s"
+ level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
app = Flask(__name__)
app.config.from_envvar("FLASK_SECRETS")
app.secret_key = app.config["SECRET_KEY"]
-HEALTHY_BLOCK_DIFF = 500 # idc to config this. hardcode is fine.
+app.register_blueprint(meta.bp)
+app.register_blueprint(api.bp)
+app.register_blueprint(cli.bp)
+app.register_blueprint(filters.bp)
-@app.route("/", methods=["GET", "POST"])
-def index():
- form = SubmitNode()
- nettype = request.args.get("network", "mainnet")
- crypto = request.args.get("chain", "monero")
- onion = request.args.get("onion", False)
- show_all = "true" == request.args.get("all", "false")
- web_compatible = request.args.get("cors", False)
- highest_block = get_highest_block(nettype, crypto)
- healthy_block = highest_block - HEALTHY_BLOCK_DIFF
-
- nodes = Node.select().where(
- Node.validated == True,
- Node.nettype == nettype,
- Node.crypto == crypto
- )
-
- if web_compatible:
- nodes = nodes.where(Node.web_compatible == True)
-
- nodes_all = nodes.count()
- nodes_unhealthy = nodes.where(
- (Node.available == False) | (Node.last_height < healthy_block)
- ).count()
-
- if not show_all:
- nodes = nodes.where(
- Node.available == True,
- Node.last_height > healthy_block
- )
-
- nodes = nodes.order_by(
- Node.datetime_entered.desc()
- )
- if onion:
- nodes = nodes.where(Node.is_tor == True)
-
- nodes = [n for n in nodes]
- shuffle(nodes)
-
- return render_template(
- "index.html",
- nodes=nodes,
- nodes_all=nodes_all,
- nodes_unhealthy=nodes_unhealthy,
- nettype=nettype,
- crypto=crypto,
- form=form,
- web_compatible=web_compatible
- )
-
-@app.route("/nodes.json")
-def nodes_json():
- nodes = Node.select().where(
- Node.validated==True
- ).where(
- Node.nettype=="mainnet"
- )
- xmr_nodes = [n for n in nodes if n.crypto == "monero"]
- wow_nodes = [n for n in nodes if n.crypto == "wownero"]
- return jsonify({
- "monero": {
- "clear": [n.url for n in xmr_nodes if n.is_tor == False],
- "onion": [n.url for n in xmr_nodes if n.is_tor == True],
- "web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
- },
- "wownero": {
- "clear": [n.url for n in wow_nodes if n.is_tor == False],
- "onion": [n.url for n in wow_nodes if n.is_tor == True]
- }
- })
-
-@app.route("/health.json")
-def health_json():
- data = {}
- nodes = Node.select().where(
- Node.validated == True
- )
- for node in nodes:
- if node.crypto not in data:
- data[node.crypto] = {}
- _d = {
- "available": node.available,
- "last_height": node.last_height,
- "datetime_entered": node.datetime_entered,
- "datetime_checked": node.datetime_checked,
- "datetime_failed": node.datetime_failed,
- "checks": [c.health for c in node.get_all_checks()]
- }
- nettype = "clear"
- if node.is_tor:
- nettype = "onion"
- elif node.web_compatible:
- if "web_compatible" not in data[node.crypto]:
- data[node.crypto]["web_compatible"] = {}
- data[node.crypto]["web_compatible"][node.url] = _d
- if nettype not in data[node.crypto]:
- data[node.crypto][nettype] = {}
- data[node.crypto][nettype][node.url] = _d
- return jsonify(data)
-
-@app.route("/haproxy.cfg")
-def haproxy():
- crypto = request.args.get('chain') or 'monero'
- nettype = request.args.get('network') or 'mainnet'
- cors = request.args.get('cors') or False
- tor = request.args.get('onion') or False
- nodes = Node.select().where(
- Node.validated == True,
- Node.nettype == nettype,
- Node.crypto == crypto,
- Node.is_tor == tor,
- Node.web_compatible == cors
- )
- tpl = render_template("haproxy.html", nodes=nodes)
- print(tpl)
- res = Response(tpl)
- res.headers['Content-Disposition'] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
- return res
-
-@app.route("/wow_nodes.json")
-def wow_nodes_json():
- nodes = Node.select().where(
- Node.validated==True
- ).where(
- Node.nettype=="mainnet"
- ).where(
- Node.crypto=="wownero"
- )
- nodes = [n for n in nodes]
- return jsonify({
- "clear": [n.url for n in nodes if n.is_tor == False],
- "onion": [n.url for n in nodes if n.is_tor == True]
- })
-
-@app.route("/map")
-def map():
- try:
- peers = rw_cache('map_peers')
- except:
- flash('Couldn\'t load the map. Try again later.')
- return redirect('/')
- return render_template(
- "map.html",
- peers=peers,
- source_node=config.NODE_HOST
- )
-
-@app.route("/resources")
-def resources():
- return render_template("resources.html")
-
-@app.route("/add", methods=["GET", "POST"])
-def add():
- if request.method == "POST":
- url = request.form.get("node_url")
- regex = re.compile(
- r"^(?:http)s?://" # http:// or https://
- r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" #domain...
- r"localhost|" #localhost...
- r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip
- r"(?::\d+)?" # optional port
- r"(?:/?|[/?]\S+)$", re.IGNORECASE
- )
- re_match = re.match(regex, url)
- if re_match is None:
- flash("This doesn\"t look like a valid URL")
- else:
- _url = urlparse(url)
- url = f"{_url.scheme}://{_url.netloc}".lower()
- if Node.select().where(Node.url == url).exists():
- flash("This node is already in the database.")
- else:
- flash("Seems like a valid node URL. Added to the database and will check soon.")
- node = Node(url=url)
- node.save()
- return redirect("/")
-
-def cleanup_health_checks():
- diff = datetime.utcnow() - timedelta(hours=24)
- checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
- for check in checks:
- print("Deleting check", check.id)
- check.delete_instance()
-
-@app.cli.command("check")
-def check():
- cleanup_health_checks()
- nodes = Node.select().where(Node.validated == True)
- for node in nodes:
- now = datetime.utcnow()
- hc = HealthCheck(node=node)
- logging.info(f"Attempting to check {node.url}")
- try:
- r = make_request(node.url)
- assert "status" in r.json()
- assert "offline" in r.json()
- assert "height" in r.json()
- has_cors = 'Access-Control-Allow-Origin' in r.headers
- is_ssl = node.url.startswith('https://')
- if r.json()["status"] == "OK":
- node.web_compatible = has_cors and is_ssl
- node.last_height = r.json()["height"]
- hc.health = True
- highest_block = get_highest_block(node.nettype, node.crypto)
- healthy_block = highest_block - HEALTHY_BLOCK_DIFF
- if r.json()["height"] < healthy_block:
- node.available = False
- logging.info("unhealthy")
- else:
- node.available = True
- logging.info("success")
- else:
- raise
- except:
- logging.info("fail")
- node.datetime_failed = now
- node.available = False
- hc.health = False
- finally:
- node.datetime_checked = now
- node.save()
- hc.save()
- if node.get_failed_checks().count() == node.get_all_checks().count() and node.get_all_checks().count() > 5:
- print('this node fails all of its health checks - deleting it!')
- for _hc in node.get_all_checks():
- _hc.delete_instance()
- node.delete_instance()
-
-@app.cli.command("get_peers")
-def get_peers():
- all_peers = []
- print('[+] Preparing to crawl Monero p2p network')
- print(f'[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
- initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
- with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
- for peer in initial_peers:
- if peer not in all_peers:
- all_peers.append(peer)
- _url = urlparse(peer)
- url = f"{_url.scheme}://{_url.netloc}".lower()
- if not Peer.select().where(Peer.url == peer).exists():
- response = reader.city(_url.hostname)
- p = Peer(
- url=peer,
- country=response.country.name,
- city=response.city.name,
- postal=response.postal.code,
- lat=response.location.latitude,
- lon=response.location.longitude,
- )
- p.save()
- print(f'{peer} - saving new peer')
- else:
- p = Peer.select().where(Peer.url == peer).first()
- p.datetime = datetime.now()
- p.save()
-
- try:
- print(f'[.] Retrieving crawled peers from {_url.netloc}')
- new_peers = retrieve_peers(_url.hostname, _url.port)
- for peer in new_peers:
- if peer not in all_peers:
- all_peers.append(peer)
- _url = urlparse(peer)
- url = f"{_url.scheme}://{_url.netloc}".lower()
- if not Peer.select().where(Peer.url == peer).exists():
- response = reader.city(_url.hostname)
- p = Peer(
- url=peer,
- country=response.country.name,
- city=response.city.name,
- postal=response.postal.code,
- lat=response.location.latitude,
- lon=response.location.longitude,
- )
- p.save()
- print(f'{peer} - saving new peer')
- else:
- p = Peer.select().where(Peer.url == peer).first()
- p.datetime = datetime.now()
- p.save()
- except:
- pass
-
- print(f'[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}')
- print('[+] Deleting old Monero p2p peers')
- for p in Peer.select():
- if p.hours_elapsed() > 24:
- print(f'[.] Deleting {p.url}')
- p.delete_instance()
- rw_cache('map_peers', list(Peer.select().execute()))
-
-
-@app.cli.command('init')
-def init():
- pass
-
-@app.cli.command("validate")
-def validate():
- nodes = Node.select().where(Node.validated == False)
- for node in nodes:
- now = datetime.utcnow()
- logging.info(f"Attempting to validate {node.url}")
- try:
- r = make_request(node.url)
- assert "height" in r.json()
- assert "nettype" in r.json()
- has_cors = 'Access-Control-Allow-Origin' in r.headers
- is_ssl = node.url.startswith('https://')
- nettype = r.json()["nettype"]
- crypto = determine_crypto(node.url)
- logging.info("success")
- if nettype in ["mainnet", "stagenet", "testnet"]:
- node.nettype = nettype
- node.available = True
- node.validated = True
- node.web_compatible = has_cors and is_ssl
- node.last_height = r.json()["height"]
- node.datetime_checked = now
- node.crypto = crypto
- node.is_tor = is_onion(node.url)
- node.save()
- else:
- logging.info("unexpected nettype")
- except requests.exceptions.ConnectTimeout:
- logging.info("connection timed out")
- node.delete_instance()
- except requests.exceptions.SSLError:
- logging.info("invalid certificate")
- node.delete_instance()
- except requests.exceptions.ConnectionError:
- logging.info("connection error")
- node.delete_instance()
- except requests.exceptions.HTTPError:
- logging.info("http error, 4xx or 5xx")
- node.delete_instance()
- except Exception as e:
- logging.info("failed for reasons unknown")
- node.delete_instance()
-
-@app.cli.command("export")
-def export():
- all_nodes = []
- ts = int(arrow.get().timestamp())
- export_dir = f"{config.DATA_DIR}/export.txt"
- export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
- nodes = Node.select().where(Node.validated == True)
- for node in nodes:
- logging.info(f"Adding {node.url}")
- all_nodes.append(node.url)
- with open(export_dir, "w") as f:
- f.write("\n".join(all_nodes))
- with open(export_dir_stamped, "w") as f:
- f.write("\n".join(all_nodes))
- logging.info(f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}")
-
-@app.cli.command("import")
-def import_():
- all_nodes = []
- export_dir = f"{config.DATA_DIR}/export.txt"
- with open(export_dir, "r") as f:
- for url in f.readlines():
- try:
- n = url.rstrip().lower()
- logging.info(f"Adding {n}")
- node = Node(url=n)
- node.save()
- all_nodes.append(n)
- except:
- pass
- logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")
-
-@app.template_filter("humanize")
-def humanize(d):
- t = arrow.get(d, "UTC")
- return t.humanize()
-
-@app.template_filter("hours_elapsed")
-def hours_elapsed(d):
- now = datetime.utcnow()
- diff = now - d
- return diff.total_seconds() / 60 / 60
-
-@app.template_filter("pop_arg")
-def trim_arg(all_args, arg_to_trim):
- d = all_args.to_dict()
- d.pop(arg_to_trim)
- return urlencode(d)
if __name__ == "__main__":
app.run()
diff --git a/xmrnodes/cli.py b/xmrnodes/cli.py
new file mode 100644
index 0000000..3d633d9
--- /dev/null
+++ b/xmrnodes/cli.py
@@ -0,0 +1,219 @@
+import logging
+from datetime import datetime, timedelta
+
+import geoip2.database
+import arrow
+import requests
+from flask import Blueprint
+from urllib.parse import urlparse
+
+from xmrnodes.helpers import determine_crypto, is_onion, make_request
+from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
+from xmrnodes.models import Node, HealthCheck, Peer
+from xmrnodes import config
+
+bp = Blueprint("cli", "cli", cli_group=None)
+
+
+@bp.cli.command("init")
+def init():
+ pass
+
+
+@bp.cli.command("check")
+def check():
+ diff = datetime.utcnow() - timedelta(hours=24)
+ checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
+ for check in checks:
+ print("Deleting check", check.id)
+ check.delete_instance()
+ nodes = Node.select().where(Node.validated == True)
+ for node in nodes:
+ now = datetime.utcnow()
+ hc = HealthCheck(node=node)
+ logging.info(f"Attempting to check {node.url}")
+ try:
+ r = make_request(node.url)
+ assert "status" in r.json()
+ assert "offline" in r.json()
+ assert "height" in r.json()
+ has_cors = "Access-Control-Allow-Origin" in r.headers
+ is_ssl = node.url.startswith("https://")
+ if r.json()["status"] == "OK":
+ node.web_compatible = has_cors and is_ssl
+ node.last_height = r.json()["height"]
+ hc.health = True
+ highest_block = get_highest_block(node.nettype, node.crypto)
+ healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
+ if r.json()["height"] < healthy_block:
+ node.available = False
+ logging.info("unhealthy")
+ else:
+ node.available = True
+ logging.info("success")
+ else:
+ raise
+ except:
+ logging.info("fail")
+ node.datetime_failed = now
+ node.available = False
+ hc.health = False
+ finally:
+ node.datetime_checked = now
+ node.save()
+ hc.save()
+ if (
+ node.get_failed_checks().count() == node.get_all_checks().count()
+ and node.get_all_checks().count() > 5
+ ):
+ print("this node fails all of its health checks - deleting it!")
+ for _hc in node.get_all_checks():
+ _hc.delete_instance()
+ node.delete_instance()
+
+
+@bp.cli.command("get_peers")
+def get_peers():
+ all_peers = []
+ print("[+] Preparing to crawl Monero p2p network")
+ print(f"[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}")
+ initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
+ with geoip2.database.Reader("./data/GeoLite2-City.mmdb") as reader:
+ for peer in initial_peers:
+ if peer not in all_peers:
+ all_peers.append(peer)
+ _url = urlparse(peer)
+ url = f"{_url.scheme}://{_url.netloc}".lower()
+ if not Peer.select().where(Peer.url == peer).exists():
+ response = reader.city(_url.hostname)
+ p = Peer(
+ url=peer,
+ country=response.country.name,
+ city=response.city.name,
+ postal=response.postal.code,
+ lat=response.location.latitude,
+ lon=response.location.longitude,
+ )
+ p.save()
+ print(f"{peer} - saving new peer")
+ else:
+ p = Peer.select().where(Peer.url == peer).first()
+ p.datetime = datetime.now()
+ p.save()
+
+ try:
+ print(f"[.] Retrieving crawled peers from {_url.netloc}")
+ new_peers = retrieve_peers(_url.hostname, _url.port)
+ for peer in new_peers:
+ if peer not in all_peers:
+ all_peers.append(peer)
+ _url = urlparse(peer)
+ url = f"{_url.scheme}://{_url.netloc}".lower()
+ if not Peer.select().where(Peer.url == peer).exists():
+ response = reader.city(_url.hostname)
+ p = Peer(
+ url=peer,
+ country=response.country.name,
+ city=response.city.name,
+ postal=response.postal.code,
+ lat=response.location.latitude,
+ lon=response.location.longitude,
+ )
+ p.save()
+ print(f"{peer} - saving new peer")
+ else:
+ p = Peer.select().where(Peer.url == peer).first()
+ p.datetime = datetime.now()
+ p.save()
+ except:
+ pass
+
+ print(
+ f"[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}"
+ )
+ print("[+] Deleting old Monero p2p peers")
+ for p in Peer.select():
+ if p.hours_elapsed() > 24:
+ print(f"[.] Deleting {p.url}")
+ p.delete_instance()
+ rw_cache("map_peers", list(Peer.select().execute()))
+
+
+@bp.cli.command("validate")
+def validate():
+ nodes = Node.select().where(Node.validated == False)
+ for node in nodes:
+ now = datetime.utcnow()
+ logging.info(f"Attempting to validate {node.url}")
+ try:
+ r = make_request(node.url)
+ assert "height" in r.json()
+ assert "nettype" in r.json()
+ has_cors = "Access-Control-Allow-Origin" in r.headers
+ is_ssl = node.url.startswith("https://")
+ nettype = r.json()["nettype"]
+ crypto = determine_crypto(node.url)
+ logging.info("success")
+ if nettype in ["mainnet", "stagenet", "testnet"]:
+ node.nettype = nettype
+ node.available = True
+ node.validated = True
+ node.web_compatible = has_cors and is_ssl
+ node.last_height = r.json()["height"]
+ node.datetime_checked = now
+ node.crypto = crypto
+ node.is_tor = is_onion(node.url)
+ node.save()
+ else:
+ logging.info("unexpected nettype")
+ except requests.exceptions.ConnectTimeout:
+ logging.info("connection timed out")
+ node.delete_instance()
+ except requests.exceptions.SSLError:
+ logging.info("invalid certificate")
+ node.delete_instance()
+ except requests.exceptions.ConnectionError:
+ logging.info("connection error")
+ node.delete_instance()
+ except requests.exceptions.HTTPError:
+ logging.info("http error, 4xx or 5xx")
+ node.delete_instance()
+ except Exception as e:
+ logging.info("failed for reasons unknown")
+ node.delete_instance()
+
+
+@bp.cli.command("export")
+def export():
+ all_nodes = []
+ ts = int(arrow.get().timestamp())
+ export_dir = f"{config.DATA_DIR}/export.txt"
+ export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
+ nodes = Node.select().where(Node.validated == True)
+ for node in nodes:
+ logging.info(f"Adding {node.url}")
+ all_nodes.append(node.url)
+ with open(export_dir, "w") as f:
+ f.write("\n".join(all_nodes))
+ with open(export_dir_stamped, "w") as f:
+ f.write("\n".join(all_nodes))
+ logging.info(
+ f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}"
+ )
+
+
+@bp.cli.command("import")
+def import_():
+ all_nodes = []
+ export_dir = f"{config.DATA_DIR}/export.txt"
+ with open(export_dir, "r") as f:
+ for url in f.readlines():
+ try:
+ n = url.rstrip().lower()
+ logging.info(f"Adding {n}")
+ node = Node(url=n)
+ node.save()
+ all_nodes.append(n)
+ except:
+ pass
+ logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")
diff --git a/xmrnodes/config.example.py b/xmrnodes/config.example.py
deleted file mode 100644
index daa0ca7..0000000
--- a/xmrnodes/config.example.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import os
-
-SECRET_KEY = os.environ.get('SECRET_KEY', 'xxxx')
-DATA_DIR = os.environ.get('DATA_DIR', './data')
-TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
-TOR_PORT = os.environ.get('TOR_PORT', 9050)
-NODE_HOST = os.environ.get('NODE_HOST', '127.0.0.1')
-NODE_PORT = os.environ.get('NODE_PORT', 18080)
diff --git a/xmrnodes/config.py b/xmrnodes/config.py
index ef42adc..ce3cafa 100644
--- a/xmrnodes/config.py
+++ b/xmrnodes/config.py
@@ -1,14 +1,15 @@
-import os
+from os import environ
from secrets import token_urlsafe
from dotenv import load_dotenv
load_dotenv()
-SECRET_KEY = os.environ.get('SECRET_KEY', token_urlsafe(14))
-SERVER_NAME = os.environ.get('SERVER_NAME', '127.0.0.1:5000')
-DATA_DIR = os.environ.get('DATA_DIR', './data')
-TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
-TOR_PORT = os.environ.get('TOR_PORT', 9050)
-NODE_HOST = os.environ.get('NODE_HOST', 'singapore.node.xmr.pm')
-NODE_PORT = os.environ.get('NODE_PORT', 18080)
\ No newline at end of file
+SECRET_KEY = environ.get("SECRET_KEY", token_urlsafe(14))
+SERVER_NAME = environ.get("SERVER_NAME", "127.0.0.1:5000")
+DATA_DIR = environ.get("DATA_DIR", "./data")
+TOR_HOST = environ.get("TOR_HOST", "127.0.0.1")
+TOR_PORT = environ.get("TOR_PORT", 9050)
+NODE_HOST = environ.get("NODE_HOST", "singapore.node.xmr.pm")
+NODE_PORT = environ.get("NODE_PORT", 18080)
+HEALTHY_BLOCK_DIFF = int(environ.get("HEALTHY_BLOCK_DIFF", 500))
diff --git a/xmrnodes/filters.py b/xmrnodes/filters.py
new file mode 100644
index 0000000..0378200
--- /dev/null
+++ b/xmrnodes/filters.py
@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import arrow
+from flask import Blueprint
+from urllib.parse import urlencode
+
+bp = Blueprint("filters", "filters")
+
+
+@bp.app_template_filter("humanize")
+def humanize(d):
+ t = arrow.get(d, "UTC")
+ return t.humanize()
+
+
+@bp.app_template_filter("hours_elapsed")
+def hours_elapsed(d):
+ now = datetime.utcnow()
+ diff = now - d
+ return diff.total_seconds() / 60 / 60
+
+
+@bp.app_template_filter("pop_arg")
+def trim_arg(all_args, arg_to_trim):
+ d = all_args.to_dict()
+ d.pop(arg_to_trim)
+ return urlencode(d)
diff --git a/xmrnodes/forms.py b/xmrnodes/forms.py
index 25b687c..861e4f0 100644
--- a/xmrnodes/forms.py
+++ b/xmrnodes/forms.py
@@ -4,4 +4,8 @@ from wtforms.validators import DataRequired
class SubmitNode(FlaskForm):
- node_url = StringField('', validators=[DataRequired()], render_kw={"placeholder": "Node URL (http://xxx.tld:18081)"})
+ node_url = StringField(
+ "",
+ validators=[DataRequired()],
+ render_kw={"placeholder": "Node URL (http://xxx.tld:18081)"},
+ )
diff --git a/xmrnodes/helpers.py b/xmrnodes/helpers.py
index cee12ce..689f27e 100644
--- a/xmrnodes/helpers.py
+++ b/xmrnodes/helpers.py
@@ -22,22 +22,30 @@ def make_request(url: str, path="/get_info", data=None):
else:
proxies = None
timeout = 10
- r = r_get(url + path, timeout=timeout, proxies=proxies, json=data, headers=headers, verify=False)
+ r = r_get(
+ url + path,
+ timeout=timeout,
+ proxies=proxies,
+ json=data,
+ headers=headers,
+ verify=False,
+ )
r.raise_for_status()
return r
+
def determine_crypto(url):
data = {"method": "get_block_header_by_height", "params": {"height": 0}}
hashes = {
"monero": [
- "418015bb9ae982a1975da7d79277c2705727a56894ba0fb246adaabb1f4632e3", #mainnet
- "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b", #testnet
- "76ee3cc98646292206cd3e86f74d88b4dcc1d937088645e9b0cbca84b7ce74eb" #stagenet
+ "418015bb9ae982a1975da7d79277c2705727a56894ba0fb246adaabb1f4632e3", # mainnet
+ "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b", # testnet
+ "76ee3cc98646292206cd3e86f74d88b4dcc1d937088645e9b0cbca84b7ce74eb", # stagenet
],
"wownero": [
- "a3fd635dd5cb55700317783469ba749b5259f0eeac2420ab2c27eb3ff5ffdc5c", #mainnet
- "d81a24c7aad4628e5c9129f8f2ec85888885b28cf468597a9762c3945e9f29aa", #testnet
- ]
+ "a3fd635dd5cb55700317783469ba749b5259f0eeac2420ab2c27eb3ff5ffdc5c", # mainnet
+ "d81a24c7aad4628e5c9129f8f2ec85888885b28cf468597a9762c3945e9f29aa", # testnet
+ ],
}
try:
r = make_request(url, "/json_rpc", data)
@@ -52,6 +60,7 @@ def determine_crypto(url):
except:
return "unknown"
+
def is_onion(url: str):
_split = url.split(":")
if len(_split) < 2:
@@ -61,21 +70,23 @@ def is_onion(url: str):
else:
return False
+
# Use hacky filesystem cache since i dont feel like shipping redis
def rw_cache(key_name, data=None):
- pickle_file = path.join(config.DATA_DIR, f'{key_name}.pkl')
+ pickle_file = path.join(config.DATA_DIR, f"{key_name}.pkl")
if data:
- with open(pickle_file, 'wb') as f:
+ with open(pickle_file, "wb") as f:
f.write(pickle.dumps(data))
return data
else:
- with open(pickle_file, 'rb') as f:
+ with open(pickle_file, "rb") as f:
pickled_data = pickle.load(f)
return pickled_data
+
def retrieve_peers(host, port):
try:
- print(f'[.] Connecting to {host}:{port}')
+ print(f"[.] Connecting to {host}:{port}")
sock = socket.socket()
sock.settimeout(5)
sock.connect((host, int(port)))
@@ -109,7 +120,7 @@ def retrieve_peers(host, port):
for peer in _peers:
try:
- peers.append('http://%s:%d' % (peer['ip'].ip, peer['port'].value))
+ peers.append("http://%s:%d" % (peer["ip"].ip, peer["port"].value))
except:
pass
@@ -121,12 +132,15 @@ def retrieve_peers(host, port):
else:
return None
+
def get_highest_block(nettype, crypto):
- highest = Node.select().where(
- Node.validated == True,
- Node.nettype == nettype,
- Node.crypto == crypto
- ).order_by(Node.last_height.desc()).limit(1).first()
+ highest = (
+ Node.select()
+ .where(Node.validated == True, Node.nettype == nettype, Node.crypto == crypto)
+ .order_by(Node.last_height.desc())
+ .limit(1)
+ .first()
+ )
if highest:
return highest.last_height
else:
diff --git a/xmrnodes/models.py b/xmrnodes/models.py
index 6de16ef..5f45cd9 100644
--- a/xmrnodes/models.py
+++ b/xmrnodes/models.py
@@ -9,6 +9,7 @@ from xmrnodes import config
db = SqliteQueueDatabase(f"{config.DATA_DIR}/sqlite.db")
+
class Node(Model):
id = AutoField()
url = CharField(unique=True)
@@ -29,7 +30,9 @@ class Node(Model):
return _url.netloc
def get_failed_checks(self):
- hcs = HealthCheck.select().where(HealthCheck.node == self, HealthCheck.health == False)
+ hcs = HealthCheck.select().where(
+ HealthCheck.node == self, HealthCheck.health == False
+ )
return hcs
def get_all_checks(self):
@@ -39,6 +42,7 @@ class Node(Model):
class Meta:
database = db
+
class Peer(Model):
id = AutoField()
url = CharField(unique=True)
@@ -60,13 +64,15 @@ class Peer(Model):
class Meta:
database = db
+
class HealthCheck(Model):
id = AutoField()
- node = ForeignKeyField(Node, backref='healthchecks')
+ node = ForeignKeyField(Node, backref="healthchecks")
datetime = DateTimeField(default=datetime.utcnow)
health = BooleanField()
class Meta:
database = db
+
db.create_tables([Node, HealthCheck, Peer])
diff --git a/xmrnodes/routes/api.py b/xmrnodes/routes/api.py
new file mode 100644
index 0000000..e74d659
--- /dev/null
+++ b/xmrnodes/routes/api.py
@@ -0,0 +1,71 @@
+from flask import jsonify, Blueprint
+
+from xmrnodes.models import Node
+
+
+bp = Blueprint('api', 'api')
+
+@bp.route("/nodes.json")
+def nodes_json():
+ nodes = Node.select().where(
+ Node.validated==True
+ ).where(
+ Node.nettype=="mainnet"
+ )
+ xmr_nodes = [n for n in nodes if n.crypto == "monero"]
+ wow_nodes = [n for n in nodes if n.crypto == "wownero"]
+ return jsonify({
+ "monero": {
+ "clear": [n.url for n in xmr_nodes if n.is_tor == False],
+ "onion": [n.url for n in xmr_nodes if n.is_tor == True],
+ "web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
+ },
+ "wownero": {
+ "clear": [n.url for n in wow_nodes if n.is_tor == False],
+ "onion": [n.url for n in wow_nodes if n.is_tor == True]
+ }
+ })
+
+@bp.route("/health.json")
+def health_json():
+ data = {}
+ nodes = Node.select().where(
+ Node.validated == True
+ )
+ for node in nodes:
+ if node.crypto not in data:
+ data[node.crypto] = {}
+ _d = {
+ "available": node.available,
+ "last_height": node.last_height,
+ "datetime_entered": node.datetime_entered,
+ "datetime_checked": node.datetime_checked,
+ "datetime_failed": node.datetime_failed,
+ "checks": [c.health for c in node.get_all_checks()]
+ }
+ nettype = "clear"
+ if node.is_tor:
+ nettype = "onion"
+ elif node.web_compatible:
+ if "web_compatible" not in data[node.crypto]:
+ data[node.crypto]["web_compatible"] = {}
+ data[node.crypto]["web_compatible"][node.url] = _d
+ if nettype not in data[node.crypto]:
+ data[node.crypto][nettype] = {}
+ data[node.crypto][nettype][node.url] = _d
+ return jsonify(data)
+
+@bp.route("/wow_nodes.json")
+def wow_nodes_json():
+ nodes = Node.select().where(
+ Node.validated==True
+ ).where(
+ Node.nettype=="mainnet"
+ ).where(
+ Node.crypto=="wownero"
+ )
+ nodes = [n for n in nodes]
+ return jsonify({
+ "clear": [n.url for n in nodes if n.is_tor == False],
+ "onion": [n.url for n in nodes if n.is_tor == True]
+ })
diff --git a/xmrnodes/routes/meta.py b/xmrnodes/routes/meta.py
new file mode 100644
index 0000000..900478f
--- /dev/null
+++ b/xmrnodes/routes/meta.py
@@ -0,0 +1,124 @@
+import re
+from random import shuffle
+
+from flask import request, redirect, Blueprint
+from flask import render_template, flash, Response
+from urllib.parse import urlparse
+
+from xmrnodes.helpers import rw_cache, get_highest_block
+from xmrnodes.forms import SubmitNode
+from xmrnodes.models import Node
+from xmrnodes import config
+
+bp = Blueprint("meta", "meta")
+
+
+@bp.route("/", methods=["GET", "POST"])
+def index():
+ form = SubmitNode()
+ nettype = request.args.get("network", "mainnet")
+ crypto = request.args.get("chain", "monero")
+ onion = request.args.get("onion", False)
+ show_all = "true" == request.args.get("all", "false")
+ web_compatible = request.args.get("cors", False)
+ highest_block = get_highest_block(nettype, crypto)
+ healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
+
+ nodes = Node.select().where(
+ Node.validated == True, Node.nettype == nettype, Node.crypto == crypto
+ )
+
+ if web_compatible:
+ nodes = nodes.where(Node.web_compatible == True)
+
+ nodes_all = nodes.count()
+ nodes_unhealthy = nodes.where(
+ (Node.available == False) | (Node.last_height < healthy_block)
+ ).count()
+
+ if not show_all:
+ nodes = nodes.where(Node.available == True, Node.last_height > healthy_block)
+
+ nodes = nodes.order_by(Node.datetime_entered.desc())
+ if onion:
+ nodes = nodes.where(Node.is_tor == True)
+
+ nodes = [n for n in nodes]
+ shuffle(nodes)
+
+ return render_template(
+ "index.html",
+ nodes=nodes,
+ nodes_all=nodes_all,
+ nodes_unhealthy=nodes_unhealthy,
+ nettype=nettype,
+ crypto=crypto,
+ form=form,
+ web_compatible=web_compatible,
+ )
+
+
+@bp.route("/map")
+def map():
+ try:
+ peers = rw_cache("map_peers")
+ except:
+ flash("Couldn't load the map. Try again later.")
+ return redirect("/")
+ return render_template("map.html", peers=peers, source_node=config.NODE_HOST)
+
+
+@bp.route("/resources")
+def resources():
+ return render_template("resources.html")
+
+
+@bp.route("/add", methods=["GET", "POST"])
+def add():
+ if request.method == "POST":
+ url = request.form.get("node_url")
+ regex = re.compile(
+ r"^(?:http)s?://" # http:// or https://
+ r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" # domain...
+ r"localhost|" # localhost...
+ r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip
+ r"(?::\d+)?" # optional port
+ r"(?:/?|[/?]\S+)$",
+ re.IGNORECASE,
+ )
+ re_match = re.match(regex, url)
+ if re_match is None:
+ flash("This doesn't look like a valid URL")
+ else:
+ _url = urlparse(url)
+ url = f"{_url.scheme}://{_url.netloc}".lower()
+ if Node.select().where(Node.url == url).exists():
+ flash("This node is already in the database.")
+ else:
+ flash(
+ "Seems like a valid node URL. Added to the database and will check soon."
+ )
+ node = Node(url=url)
+ node.save()
+ return redirect("/")
+
+
+@bp.route("/haproxy.cfg")
+def haproxy():
+ crypto = request.args.get("chain") or "monero"
+ nettype = request.args.get("network") or "mainnet"
+ cors = request.args.get("cors") or False
+ tor = request.args.get("onion") or False
+ nodes = Node.select().where(
+ Node.validated == True,
+ Node.nettype == nettype,
+ Node.crypto == crypto,
+ Node.is_tor == tor,
+ Node.web_compatible == cors,
+ )
+ tpl = render_template("haproxy.html", nodes=nodes)
+ res = Response(tpl)
+ res.headers[
+ "Content-Disposition"
+ ] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
+ return res
diff --git a/xmrnodes/static/css/style.css b/xmrnodes/static/css/style.css
index 84bd72c..b69239f 100644
--- a/xmrnodes/static/css/style.css
+++ b/xmrnodes/static/css/style.css
@@ -138,4 +138,4 @@ input[type="text"] {
td img {
padding-right: .75em;
-}
\ No newline at end of file
+}
diff --git a/xmrnodes/templates/base.html b/xmrnodes/templates/base.html
index 7ed5d8a..9699268 100644
--- a/xmrnodes/templates/base.html
+++ b/xmrnodes/templates/base.html
@@ -43,13 +43,15 @@
diff --git a/xmrnodes/templates/index.html b/xmrnodes/templates/index.html
index a0660ce..de955b1 100644
--- a/xmrnodes/templates/index.html
+++ b/xmrnodes/templates/index.html
@@ -9,7 +9,7 @@
Add A Node
-