Merge branch 'main' of github.com:lalanza808/monero.fail into levin

Makefile

@@ -1,3 +1,7 @@
+setup:
+	python3 -m venv .venv
+	.venv/bin/pip install -r requirements.txt
+
 up:
 	docker-compose up -d
 

requirements.txt

@@ -4,4 +4,6 @@ peewee
 gunicorn
 arrow
 flask_wtf
+pysocks
 git+https://github.com/cdiv1e12/py-levin
+geoip2

xmrnodes/app.py

@@ -1,18 +1,23 @@
-import arrow
+
 import json
-import requests
 import re
 import logging
-import click
 from os import makedirs
 from random import shuffle
+from socket import gethostbyname_ex
 from datetime import datetime, timedelta
-from flask import Flask, request, redirect
+
+import geoip2.database
+import arrow
+import requests
+import click
+from flask import Flask, request, redirect, jsonify
 from flask import render_template, flash, url_for
 from urllib.parse import urlparse
+
 from xmrnodes.helpers import determine_crypto, is_onion, make_request, retrieve_peers
 from xmrnodes.forms import SubmitNode
-from xmrnodes.models import Node, HealthCheck
+from xmrnodes.models import Node, HealthCheck, Peer
 from xmrnodes import config
 
 
@@ -54,6 +59,68 @@ def index():
         form=form
     )
 
+@app.route("/nodes.json")
+def nodes_json():
+    nodes = Node.select().where(
+        Node.validated==True
+    ).where(
+        Node.nettype=="mainnet"
+    )
+    xmr_nodes = [n for n in nodes if n.crypto == "monero"]
+    wow_nodes = [n for n in nodes if n.crypto == "wownero"]
+    return jsonify({
+        "monero": {
+            "clear": [n.url for n in xmr_nodes if n.is_tor == False],
+            "onion": [n.url for n in xmr_nodes if n.is_tor == True]
+        },
+        "wownero": {
+            "clear": [n.url for n in wow_nodes if n.is_tor == False],
+            "onion": [n.url for n in wow_nodes if n.is_tor == True]
+        }
+    })
+
+@app.route("/wow_nodes.json")
+def wow_nodes_json():
+    nodes = Node.select().where(
+        Node.validated==True
+    ).where(
+        Node.nettype=="mainnet"
+    ).where(
+        Node.crypto=="wownero"
+    )
+    nodes = [n for n in nodes]
+    return jsonify({
+        "clear": [n.url for n in nodes if n.is_tor == False],
+        "onion": [n.url for n in nodes if n.is_tor == True]
+    })
+
+@app.route("/map")
+def map():
+    peers = Peer.select()
+    nodes = list()
+    _nodes = Node.select().where(
+        Node.is_tor == False,
+        Node.crypto == 'monero',
+        Node.validated == True,
+        Node.nettype == 'mainnet'
+    )
+    with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
+        for node in _nodes:
+            try:
+                _url = urlparse(node.url)
+                ip = gethostbyname_ex(_url.hostname)[2][0]
+                response = reader.city(ip)
+                nodes.append((response.location.longitude, response.location.latitude, _url.hostname, node.datetime_entered))
+            except:
+                pass
+
+    return render_template(
+        "map.html",
+        peers=peers,
+        nodes=nodes,
+        source_node=config.NODE_HOST
+    )
+
 @app.route("/resources")
 def resources():
     return render_template("resources.html")
@@ -123,8 +190,56 @@ def check():
 
 @app.cli.command("get_peers")
 def get_peers():
-    r = retrieve_peers()
-    print(r)
+    all_peers = []
+    print(f'[+] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
+    initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
+    with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
+        for peer in initial_peers:
+            if peer not in all_peers:
+                all_peers.append(peer)
+            _url = urlparse(peer)
+            url = f"{_url.scheme}://{_url.netloc}".lower()
+            if not Peer.select().where(Peer.url == peer).exists():
+                response = reader.city(_url.hostname)
+                p = Peer(
+                    url=peer,
+                    country=response.country.name,
+                    city=response.city.name,
+                    postal=response.postal.code,
+                    lat=response.location.latitude,
+                    lon=response.location.longitude,
+                )
+                p.save()
+                print(f'{peer} - saving new peer')
+            else:
+                print(f'{peer} - already seen')
+
+            try:
+                print(f'[+] Retrieving crawled peers from {_url.netloc}')
+                new_peers = retrieve_peers(_url.hostname, _url.port)
+                for peer in new_peers:
+                    all_peers.append(peer)
+                    _url = urlparse(peer)
+                    url = f"{_url.scheme}://{_url.netloc}".lower()
+                    if not Peer.select().where(Peer.url == peer).exists():
+                        response = reader.city(_url.hostname)
+                        p = Peer(
+                            url=peer,
+                            country=response.country.name,
+                            city=response.city.name,
+                            postal=response.postal.code,
+                            lat=response.location.latitude,
+                            lon=response.location.longitude,
+                        )
+                        p.save()
+                        print(f'{peer} - saving new peer')
+                    else:
+                        print(f'{peer} - already seen')
+            except:
+                pass
+
+    print(f'{len(all_peers)} peers found from {config.NODE_HOST}:{config.NODE_PORT}')
+
 
 @app.cli.command("validate")
 def validate():
@@ -182,7 +297,7 @@ def export():
 def import_():
     all_nodes = []
     export_dir = f"{config.DATA_DIR}/export.txt"
-    with open(export_dir, 'r') as f:
+    with open(export_dir, "r") as f:
         for url in f.readlines():
             try:
                 n = url.rstrip().lower()

xmrnodes/helpers.py

@@ -54,12 +54,14 @@ def is_onion(url: str):
     else:
         return False
 
-def retrieve_peers():
+def retrieve_peers(host, port):
     try:
+        print(f'[.] Connecting to {host}:{port}')
         sock = socket.socket()
-        sock.connect((config.NODE_HOST, int(config.NODE_PORT))
+        sock.settimeout(5)
+        sock.connect((host, int(port)))
     except:
-        sys.stderr.write("unable to connect to %s:%d\n" % (config.NODE_HOST, int(config.NODE_PORT))
+        sys.stderr.write("unable to connect to %s:%d\n" % (host, int([port])))
         sys.exit()
 
     bucket = Bucket.create_handshake_request()
@@ -84,9 +86,9 @@ def retrieve_peers():
         buckets.append(bucket)
 
         if bucket.command == 1001:
-            peers = bucket.get_peers() or []
+            _peers = bucket.get_peers() or []
 
-            for peer in peers:
+            for peer in _peers:
                 try:
                     peers.append('http://%s:%d' % (peer['ip'].ip, peer['port'].value))
                 except:

xmrnodes/models.py

@@ -1,5 +1,8 @@
-from peewee import *
+from urllib.parse import urlparse
 from datetime import datetime
+
+from peewee import *
+
 from xmrnodes import config
 
 
@@ -22,6 +25,22 @@ class Node(Model):
     class Meta:
         database = db
 
+class Peer(Model):
+    id = AutoField()
+    url = CharField(unique=True)
+    country = CharField(null=True)
+    city = CharField(null=True)
+    postal = IntegerField(null=True)
+    lat = FloatField(null=True)
+    lon = FloatField(null=True)
+    datetime = DateTimeField(default=datetime.utcnow)
+
+    def get_ip(self):
+        return urlparse(self.url).hostname
+
+    class Meta:
+        database = db
+
 class HealthCheck(Model):
     id = AutoField()
     node = ForeignKeyField(Node, backref='healthchecks')
@@ -31,4 +50,4 @@ class HealthCheck(Model):
     class Meta:
         database = db
 
-db.create_tables([Node, HealthCheck])
+db.create_tables([Node, HealthCheck, Peer])

xmrnodes/templates/base.html

@@ -45,6 +45,8 @@
   <br>
   <a href="https://twitter.com/lza_menace" target="_blank">Contact me</a>
   -
+  <a href="{{ url_for('map') }}">Map</a>
+  -
   <a href="https://github.com/lalanza808/monero.fail" target="_blank">Source Code</a>
   -
   <a href="{{ url_for('resources') }}">Resources</a>

xmrnodes/templates/map.html

@@ -0,0 +1,190 @@
+<!DOCTYPE html>
+ <html lang="en">
+ <head>
+     <meta charset="UTF-8">
+     <meta http-equiv="X-UA-Compatible" content="IE=edge">
+     <meta name="viewport" content="width=device-width, initial-scale=1">
+
+     <meta name="HandheldFriendly" content="True">
+     <meta name="MobileOptimized" content="320">
+     <link rel="shortcut icon" href="/static/favicon.ico" type="image/x-icon" />
+
+     <meta property="fb:app_id" content="0" />
+     <meta property="og:image" content="https://www.getmonero.org/press-kit/symbols/monero-symbol-on-white-480.png" />
+     <meta property="og:description" content="xmrnodes" />
+     <meta property="og:url" content="http://localhost" />
+     <meta property="og:title" content="XMR Nodes" />
+     <meta property="og:type" content="website" />
+
+     <meta name="viewport" content="width=device-width, initial-scale=1"/>
+     <meta name="theme-color" content="#ffffff">
+     <meta name="apple-mobile-web-app-title" content="XMR Nodes">
+     <meta name="application-name" content="XMR Nodes">
+     <meta name="msapplication-TileColor" content="#da532c">
+     <meta name="keywords" content="wownero, monero, xmr, bitmonero, cryptocurrency">
+
+     <link href="/static/css/normalize.css" rel="stylesheet">
+     <link href="/static/css/pure.css" rel="stylesheet">
+     <link href="/static/css/style.css" rel="stylesheet">
+     <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/openlayers/openlayers.github.io@master/en/v6.5.0/css/ol.css" type="text/css">
+     <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" type="text/css">
+     <style>
+       .map {
+         height: 450px;
+         margin: 0;
+         padding: 0;
+       }
+       .popover-body {
+          min-width: 276px;
+        }
+     </style>
+     <script src="https://cdn.polyfill.io/v3/polyfill.min.js?features=fetch,requestAnimationFrame,Element.prototype.classList,URL,TextDecoder"></script>
+     <script src="//cdn.jsdelivr.net/gh/openlayers/openlayers.github.io@master/en/v6.5.0/build/ol.js"></script>
+     <script src="//code.jquery.com/jquery-3.5.1.min.js"></script>
+     <script src="//maxcdn.bootstrapcdn.com/bootstrap/4.5.0/js/bootstrap.bundle.min.js"></script>
+     <title>XMR Nodes</title>
+ </head>
+
+ <body>
+
+{% with messages = get_flashed_messages() %}
+  {% if messages %}
+    <ul class="flashes pure-u-1 center">
+    {% for message in messages %}
+      <li>{{ message }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+{% endwith %}
+
+<div id="" class="center">
+  <br>
+  <a href="https://twitter.com/lza_menace" target="_blank">Contact me</a>
+  -
+  <a href="https://github.com/lalanza808/monero.fail" target="_blank">Source Code</a>
+  -
+  <a href="{{ url_for('resources') }}">Resources</a>
+</div>
+
+ <!-- Map -->
+<h2>View Map</h2>
+<div id="map" class="map"></div>
+<div id="popup" class="popup" title="Welcome to OpenLayers"></div>
+<p>Found Peers (via source node, levin p2p): {{ peers | length }}</p>
+<p>Added Nodes (Monero mainnet): {{ nodes | length }}</p>
+<p>Source Node: {{ source_node }}</p>
+<p>
+  This is not a full representation of the entire Monero network,
+  just a look into the peers being crawled from the source node ({{ source_node }})
+  and the nodes already added to the monero.fail database.
+  New peers are searched for on a recurring interval throughout the day.
+</p>
+
+<!-- Footer -->
+<div id="footer" class="center">
+  <a href="/">Go home</a>
+</div>
+
+<script>
+ // Marker layer
+ markerLayer = new ol.layer.Vector({
+    source: new ol.source.Vector({
+      features: [],
+      projection: 'EPSG:3857'
+    })
+  });
+
+ // Create the map
+ var map = new ol.Map({
+   target: 'map',
+   layers: [
+     new ol.layer.Tile({
+       source: new ol.source.OSM()
+     }),
+     markerLayer
+   ],
+   view: new ol.View({
+     center: ol.proj.fromLonLat([0, 25]),
+     zoom: 1
+   })
+ });
+
+ // Define a circle marker
+ var circle = new ol.style.Style({
+      image: new ol.style.Circle({
+          radius: 4,
+          fill: new ol.style.Fill({
+              color: 'rgba(76,76,76,0.7)',
+          }),
+          stroke: new ol.style.Stroke({
+              color: 'rgba(53,53,53,0.7)',
+              width: 1
+          })
+      })
+  });
+  var circle2 = new ol.style.Style({
+       image: new ol.style.Circle({
+           radius: 8,
+           fill: new ol.style.Fill({
+               color: 'rgba(255,102,0,0.5)',
+           }),
+           stroke: new ol.style.Stroke({
+               color: 'rgba(204,81,0,0.5)',
+               width: 1
+           })
+       })
+   });
+
+ {% for peer in peers %}
+   var feature = new ol.Feature(
+       new ol.geom.Point(ol.proj.transform(['{{ peer.lon }}', '{{ peer.lat }}'], 'EPSG:4326', 'EPSG:3857'))
+   );
+   feature.description = [
+     'Peer {{ peer.get_ip() }}',
+     'Last seen {{ peer.datetime }}'
+   ];
+   feature.setStyle(circle);
+   markerLayer.getSource().addFeature(feature);
+ {% endfor %}
+
+ {% for peer in nodes %}
+   var feature = new ol.Feature(
+       new ol.geom.Point(ol.proj.transform(['{{ peer[0] }}', '{{ peer[1] }}'], 'EPSG:4326', 'EPSG:3857'))
+   );
+   feature.description = [
+     'Node {{ peer[2] }}',
+     'Last seen {{ peer[3] }}'
+   ];
+   feature.setStyle(circle2);
+   markerLayer.getSource().addFeature(feature);
+ {% endfor %}
+
+  // Setup popup
+  var popup = new ol.Overlay({
+    element: $('#popup')[0],
+  });
+  map.addOverlay(popup);
+
+  // Show details on each pixel
+  map.on("click", function(e) {
+      var element = popup.getElement();
+      $(element).popover('dispose')
+      map.forEachFeatureAtPixel(e.pixel, function (feature, layer) {
+          var coordinate = e.coordinate;
+          $(element).popover('dispose');
+          popup.setPosition(coordinate);
+          element.title = feature.description[0]
+          $(element).popover({
+            container: element,
+            placement: 'top',
+            animation: false,
+            html: true,
+            content: '<p>' + feature.description[1] + '</p>',
+          });
+          $(element).popover('show');
+      });
+  });
+</script>
+
+</body>
+</html>