add caching, remove nodes (only show peers on map)

xmrnodes/app.py

@@ -15,7 +15,8 @@ from flask import Flask, request, redirect, jsonify
 from flask import render_template, flash, url_for
 from urllib.parse import urlparse
 
-from xmrnodes.helpers import determine_crypto, is_onion, make_request, retrieve_peers
+from xmrnodes.helpers import determine_crypto, is_onion, make_request
+from xmrnodes.helpers import retrieve_peers, rw_cache
 from xmrnodes.forms import SubmitNode
 from xmrnodes.models import Node, HealthCheck, Peer
 from xmrnodes import config
@@ -96,28 +97,9 @@ def wow_nodes_json():
 
 @app.route("/map")
 def map():
-    peers = Peer.select()
-    nodes = list()
-    _nodes = Node.select().where(
-        Node.is_tor == False,
-        Node.crypto == 'monero',
-        Node.validated == True,
-        Node.nettype == 'mainnet'
-    )
-    with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
-        for node in _nodes:
-            try:
-                _url = urlparse(node.url)
-                ip = gethostbyname_ex(_url.hostname)[2][0]
-                response = reader.city(ip)
-                nodes.append((response.location.longitude, response.location.latitude, _url.hostname, node.datetime_entered))
-            except:
-                pass
-
     return render_template(
         "map.html",
-        peers=peers,
-        nodes=nodes,
+        peers=rw_cache('map_peers'),
         source_node=config.NODE_HOST
     )
 
@@ -239,6 +221,7 @@ def get_peers():
                 pass
 
     print(f'{len(all_peers)} peers found from {config.NODE_HOST}:{config.NODE_PORT}')
+    rw_cache('map_peers', all_peers)
 
 
 @app.cli.command("validate")

xmrnodes/helpers.py

@@ -1,10 +1,14 @@
 import sys
 import socket
+import pickle
+from os import path
+
+from requests import get as r_get
 from levin.section import Section
 from levin.bucket import Bucket
 from levin.ctypes import *
 from levin.constants import LEVIN_SIGNATURE
-from requests import get as r_get
+
 from xmrnodes import config
 
 
@@ -54,6 +58,18 @@ def is_onion(url: str):
     else:
         return False
 
+# Use hacky filesystem cache since i dont feel like shipping redis
+def rw_cache(key_name, data=None):
+    pickle_file = path.join(config.DATA_DIR, f'{key_name}.pkl')
+    if data:
+        with open(pickle_file, 'wb') as f:
+            f.write(pickle.dumps(data))
+            return data
+    else:
+        with open(pickle_file, 'rb') as f:
+            pickled_data = pickle.load(f)
+            return pickled_data
+
 def retrieve_peers(host, port):
     try:
         print(f'[.] Connecting to {host}:{port}')

xmrnodes/templates/map.html

@@ -62,7 +62,6 @@
 <div id="map" class="map"></div>
 <div id="popup" class="popup" title="Welcome to OpenLayers"></div>
 <p>Found Peers (via source node, levin p2p): {{ peers | length }}</p>
-<p>Added Nodes (Monero mainnet): {{ nodes | length }}</p>
 <p>Source Node: {{ source_node }}</p>
 <p>
   This is not a full representation of the entire Monero network,
@@ -112,53 +111,29 @@
  // Define a circle marker
  var circle = new ol.style.Style({
       image: new ol.style.Circle({
-          radius: 4,
+          radius: 5,
           fill: new ol.style.Fill({
-              color: 'rgba(76,76,76,0.7)',
+              color: 'rgba(76,76,76,.75)',
           }),
           stroke: new ol.style.Stroke({
-              color: 'rgba(53,53,53,0.7)',
+              color: 'rgba(53,53,53,.75)',
               width: 1
           })
       })
   });
-  var circle2 = new ol.style.Style({
-       image: new ol.style.Circle({
-           radius: 8,
-           fill: new ol.style.Fill({
-               color: 'rgba(255,102,0,0.5)',
-           }),
-           stroke: new ol.style.Stroke({
-               color: 'rgba(204,81,0,0.5)',
-               width: 1
-           })
-       })
-   });
 
  {% for peer in peers %}
    var feature = new ol.Feature(
        new ol.geom.Point(ol.proj.transform(['{{ peer.lon }}', '{{ peer.lat }}'], 'EPSG:4326', 'EPSG:3857'))
    );
    feature.description = [
-     'Peer {{ peer.get_ip() }}',
+     'Peer {{ peer.url }}',
      'Last seen {{ peer.datetime }}'
    ];
    feature.setStyle(circle);
    markerLayer.getSource().addFeature(feature);
  {% endfor %}
 
- {% for peer in nodes %}
-   var feature = new ol.Feature(
-       new ol.geom.Point(ol.proj.transform(['{{ peer[0] }}', '{{ peer[1] }}'], 'EPSG:4326', 'EPSG:3857'))
-   );
-   feature.description = [
-     'Node {{ peer[2] }}',
-     'Last seen {{ peer[3] }}'
-   ];
-   feature.setStyle(circle2);
-   markerLayer.getSource().addFeature(feature);
- {% endfor %}
-
   // Setup popup
   var popup = new ol.Overlay({
     element: $('#popup')[0],