import logging from datetime import datetime, timedelta from time import sleep import geoip2.database import arrow import requests from flask import Blueprint from urllib.parse import urlparse from xmrnodes.helpers import determine_crypto, is_onion, is_i2p, make_request, get_whois from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block, get_geoip from xmrnodes.models import Node, HealthCheck, Peer from xmrnodes import config bp = Blueprint("cli", "cli", cli_group=None) @bp.cli.command("init") def init(): pass @bp.cli.command("check") def check_nodes(): diff = datetime.utcnow() - timedelta(hours=96) checks = HealthCheck.select().where(HealthCheck.datetime <= diff) for check in checks: print("Deleting check", check.id) check.delete_instance() nodes = Node.select().where(Node.validated == True) for node in nodes: try: check_node(node.url) except KeyboardInterrupt: exit() def check_node(_node): if _node.startswith("http"): node = Node.select().where(Node.url == _node).first() else: node = Node.select().where(Node.id == _node).first() if not node: print('node found') pass now = datetime.utcnow() hc = HealthCheck(node=node) logging.info(f"Attempting to check {node.url}") try: r = make_request(node.url) assert "status" in r.json() assert "offline" in r.json() assert "height" in r.json() if 'donation_address' in r.json(): node.donation_address = r.json()['donation_address'] has_cors = "Access-Control-Allow-Origin" in r.headers is_ssl = node.url.startswith("https://") if r.json()["status"] == "OK": node.web_compatible = has_cors and is_ssl node.last_height = r.json()["height"] hc.health = True highest_block = get_highest_block(node.nettype, node.crypto) healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF if r.json()["height"] < healthy_block: node.available = False logging.info("unhealthy") else: node.available = True logging.info("success") else: raise except: logging.info("fail") node.datetime_failed = now node.available = False hc.health = False finally: node.datetime_checked = now node.save() hc.save() # if ( # node.get_failed_checks().count() == node.get_all_checks().count() # and node.get_all_checks().count() > 5 # ): # print("this node fails all of its health checks - deleting it!") # for _hc in node.get_all_checks(): # _hc.delete_instance() # node.delete_instance() @bp.cli.command("get_peers") def get_peers(): """ This command requests peers from the configured upstream node and fans out to recursively scrape all other peers on the network. This will take several hours to run. """ # keep track of all peers all_peers = [] print("[+] Preparing to crawl Monero p2p network") print(f"[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}") # start initial list of peers to scan peers_to_scan = retrieve_peers(config.NODE_HOST, config.NODE_PORT) print(f"[+] Found {len(peers_to_scan)} initial peers to begin scraping.") sleep(3) # helper function to add a new peer to the db or update an existing one def save_peer(peer): with geoip2.database.Reader("./data/GeoLite2-City.mmdb") as reader: _url = urlparse(peer) url = f"{_url.scheme}://{_url.netloc}".lower() # add new peer if not in db if not Peer.select().where(Peer.url == peer).exists(): response = reader.city(_url.hostname) p = Peer( url=peer, country=response.country.name, city=response.city.name, postal=response.postal.code, lat=response.location.latitude, lon=response.location.longitude, ) p.save() print(f"{peer} - saving new peer") # or update if it does else: p = Peer.select().where(Peer.url == peer).first() p.datetime = datetime.now() p.save() return _url # iterate over the whole list until all peers have been scanned # add new peers to the list # skip the peer if we've seen it already try: while peers_to_scan: _peer = peers_to_scan[0] peers_to_scan.pop(0) if _peer in all_peers: print(f'already found {_peer}') continue all_peers.append(_peer) try: peer = save_peer(_peer) peers_to_scan += retrieve_peers(peer.hostname, peer.port) except: pass except KeyboardInterrupt: print('Stopped.') print( f"[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}" ) print("[+] Deleting old Monero p2p peers") for p in Peer.select(): if p.hours_elapsed() > config.PEER_LIFETIME: print(f"[.] Deleting {p.url}") p.delete_instance() rw_cache("map_peers", list(Peer.select().execute())) @bp.cli.command("validate") def validate(): nodes = Node.select().where(Node.validated == False) for node in nodes: now = datetime.utcnow() logging.info(f"Attempting to validate {node.url}") try: r = make_request(node.url) assert "height" in r.json() assert "nettype" in r.json() has_cors = "Access-Control-Allow-Origin" in r.headers is_ssl = node.url.startswith("https://") nettype = r.json()["nettype"] crypto = determine_crypto(node.url) if nettype in ["mainnet", "stagenet", "testnet"]: node.nettype = nettype node.available = True node.validated = True node.web_compatible = has_cors and is_ssl node.last_height = r.json()["height"] node.datetime_checked = now node.crypto = crypto node.is_tor = is_onion(node.url) node.is_i2p = is_i2p(node.url) if not node.is_tor and not node.is_i2p: geoip = get_geoip(node.url) node.country_name = geoip.country.name node.country_code = geoip.country.iso_code node.city = geoip.city.name node.postal = geoip.postal.code node.lat = geoip.location.latitude node.lon = geoip.location.longitude logging.info(f"found geo data for {node.url} - {node.country_code}, {node.country_name}, {node.city}") whois = get_whois(node.url) if whois: logging.info(f"found whois data for {node.url}") node.asn = whois['asn'] node.asn_cidr = whois['asn_cidr'] node.asn_country_code = whois['asn_country_code'] node.asn_description = whois['asn_description'] node.save() logging.info("success") else: logging.info("unexpected nettype") except requests.exceptions.ConnectTimeout: logging.info("connection timed out") node.delete_instance() except requests.exceptions.SSLError: logging.info("invalid certificate") node.delete_instance() except requests.exceptions.ConnectionError: logging.info("connection error") node.delete_instance() except requests.exceptions.HTTPError: logging.info("http error, 4xx or 5xx") node.delete_instance() except Exception as e: logging.info(f"failed for reasons unknown: {e}") node.delete_instance() @bp.cli.command("export") def export(): all_nodes = [] ts = int(arrow.get().timestamp()) export_dir = f"{config.DATA_DIR}/export.txt" export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt" nodes = Node.select().where(Node.validated == True) for node in nodes: logging.info(f"Adding {node.url}") all_nodes.append(node.url) with open(export_dir, "w") as f: f.write("\n".join(all_nodes)) with open(export_dir_stamped, "w") as f: f.write("\n".join(all_nodes)) logging.info( f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}" ) @bp.cli.command("import") def import_(): all_nodes = [] export_dir = f"{config.DATA_DIR}/export.txt" with open(export_dir, "r") as f: for url in f.readlines(): try: n = url.rstrip().lower() logging.info(f"Adding {n}") node = Node(url=n) node.save() all_nodes.append(n) except: pass logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")