add pruning to crawled peers and change colors on map

xmrnodes/app.py

@@ -1,4 +1,3 @@
-
 import json
 import re
 import logging
@@ -173,7 +172,8 @@ def check():
 @app.cli.command("get_peers")
 def get_peers():
     all_peers = []
-    print(f'[+] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
+    print('[+] Preparing to crawl Monero p2p network')
+    print(f'[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
     initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
     with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
         for peer in initial_peers:
@@ -194,13 +194,16 @@ def get_peers():
                 p.save()
                 print(f'{peer} - saving new peer')
             else:
-                print(f'{peer} - already seen')
+                p = Peer.select().where(Peer.url == peer).first()
+                p.datetime = datetime.now()
+                p.save()
 
             try:
-                print(f'[+] Retrieving crawled peers from {_url.netloc}')
+                print(f'[.] Retrieving crawled peers from {_url.netloc}')
                 new_peers = retrieve_peers(_url.hostname, _url.port)
                 for peer in new_peers:
-                    all_peers.append(peer)
+                    if peer not in all_peers:
+                        all_peers.append(peer)
                     _url = urlparse(peer)
                     url = f"{_url.scheme}://{_url.netloc}".lower()
                     if not Peer.select().where(Peer.url == peer).exists():
@@ -216,12 +219,19 @@ def get_peers():
                         p.save()
                         print(f'{peer} - saving new peer')
                     else:
-                        print(f'{peer} - already seen')
+                        p = Peer.select().where(Peer.url == peer).first()
+                        p.datetime = datetime.now()
+                        p.save()
             except:
                 pass
 
-    print(f'{len(all_peers)} peers found from {config.NODE_HOST}:{config.NODE_PORT}')
+    print(f'[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}')
-    rw_cache('map_peers', all_peers)
+    print('[+] Deleting old Monero p2p peers')
+    for p in Peer.select():
+        if p.hours_elapsed() > 24:
+            print(f'[.] Deleting {p.url}')
+            p.delete_instance()
+    rw_cache('map_peers', list(Peer.select().execute()))
 
 
 @app.cli.command("validate")
@@ -297,5 +307,11 @@ def humanize(d):
     t = arrow.get(d, "UTC")
     return t.humanize()
 
+@app.template_filter("hours_elapsed")
+def hours_elapsed(d):
+    now = datetime.utcnow()
+    diff = now - d
+    return diff.total_seconds() / 60 / 60
+
 if __name__ == "__main__":
     app.run()

xmrnodes/models.py

@@ -35,6 +35,11 @@ class Peer(Model):
     lon = FloatField(null=True)
     datetime = DateTimeField(default=datetime.utcnow)
 
+    def hours_elapsed(self):
+        now = datetime.utcnow()
+        diff = now - self.datetime
+        return diff.total_seconds() / 60 / 60
+
     def get_ip(self):
         return urlparse(self.url).hostname
 

xmrnodes/templates/map.html

@@ -33,6 +33,11 @@
        .popover-body {
           min-width: 276px;
         }
+        .center.info {
+          width: 50%;
+          margin: auto;
+          padding: 2em;
+        }
      </style>
      <script src="https://cdn.polyfill.io/v3/polyfill.min.js?features=fetch,requestAnimationFrame,Element.prototype.classList,URL,TextDecoder"></script>
      <script src="//cdn.jsdelivr.net/gh/openlayers/openlayers.github.io@master/en/v6.5.0/build/ol.js"></script>
@@ -53,36 +58,26 @@
   {% endif %}
 {% endwith %}
 
-<div class="center">
+<div class="center info">
-  <a href="#footer">See more</a>
+  <p>Peers seen ~24 hours: {{ peers | length }}</p>
-</div>
+  <p>Source Node: {{ source_node }}</p>
-
+  <p>
- <!-- Map -->
+    This is not a full representation of the entire Monero network,
-<h2>View Map</h2>
+    just a look into the peers being crawled from the source node ({{ source_node }}).
-<div id="map" class="map"></div>
+    New peers are searched for on a recurring interval throughout the day.
-<div id="popup" class="popup" title="Welcome to OpenLayers"></div>
+    Older peers are shown as more transparent and will be removed if not seen again after 24 hours.
-<p>Found Peers (via source node, levin p2p): {{ peers | length }}</p>
+  </p>
-<p>Source Node: {{ source_node }}</p>
-<p>
-  This is not a full representation of the entire Monero network,
-  just a look into the peers being crawled from the source node ({{ source_node }})
-  and the nodes already added to the monero.fail database.
-  New peers are searched for on a recurring interval throughout the day.
-</p>
-
-<div id="" class="center">
   <br>
+  <a href="/">Go home</a>
+  -
   <a href="https://twitter.com/lza_menace" target="_blank">Contact me</a>
   -
   <a href="https://github.com/lalanza808/monero.fail" target="_blank">Source Code</a>
   -
   <a href="{{ url_for('resources') }}">Resources</a>
 </div>
-
+<div id="map" class="map"></div>
-<!-- Footer -->
+<div id="popup" class="popup" title="Welcome to OpenLayers"></div>
-<div id="footer" class="center">
-  <a href="/">Go home</a>
-</div>
 
 <script>
  // Marker layer
@@ -108,30 +103,40 @@
    })
  });
 
- // Define a circle marker
- var circle = new ol.style.Style({
-      image: new ol.style.Circle({
-          radius: 5,
-          fill: new ol.style.Fill({
-              color: 'rgba(76,76,76,.75)',
-          }),
-          stroke: new ol.style.Stroke({
-              color: 'rgba(53,53,53,.75)',
-              width: 1
-          })
-      })
-  });
-
  {% for peer in peers %}
+
+   {% if peer.datetime | hours_elapsed > 24 %}
+      {% set fill_color = 'rgba(238,111,45,.2)' %}
+   {% elif peer.datetime | hours_elapsed > 16 %}
+      {% set fill_color = 'rgba(238,111,45,.4)' %}
+   {% elif peer.datetime | hours_elapsed > 8 %}
+      {% set fill_color = 'rgba(238,111,45,.6)' %}
+   {% elif peer.datetime | hours_elapsed > 4 %}
+      {% set fill_color = 'rgba(238,111,45,.8)' %}
+   {% else %}
+      {% set fill_color = 'rgba(238,111,45,1)' %}
+   {% endif %}
    var feature = new ol.Feature(
        new ol.geom.Point(ol.proj.transform(['{{ peer.lon }}', '{{ peer.lat }}'], 'EPSG:4326', 'EPSG:3857'))
    );
    feature.description = [
      'Peer {{ peer.url }}',
-     'Last seen {{ peer.datetime }}'
+     'Last seen {{ peer.datetime | humanize }}'
    ];
-   feature.setStyle(circle);
+   feature.setStyle(new ol.style.Style({
-   markerLayer.getSource().addFeature(feature);
+        image: new ol.style.Circle({
+            radius: 6,
+            fill: new ol.style.Fill({
+                color: '{{ fill_color }}',
+            }),
+            stroke: new ol.style.Stroke({
+                color: '#fff',
+                width: 1
+            })
+        })
+    })
+  );
+  markerLayer.getSource().addFeature(feature);
  {% endfor %}
 
   // Setup popup