From b635e39e046d1bb67f561ed4af979c57afa82aaa Mon Sep 17 00:00:00 2001 From: lza_menace Date: Fri, 2 Sep 2022 13:29:54 -0700 Subject: [PATCH] prepping migration to new database --- export.py | 72 ++++++++++++++++++++ import.py | 100 ++++++++++++++++++++++++++++ suchwow/_models.py | 159 +++++++++++++++++++++++++++++++++++++++++++++ suchwow/cli.py | 4 +- suchwow/wownero.py | 1 - 5 files changed, 333 insertions(+), 3 deletions(-) create mode 100755 export.py create mode 100755 import.py create mode 100644 suchwow/_models.py diff --git a/export.py b/export.py new file mode 100755 index 0000000..8d7463a --- /dev/null +++ b/export.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +# export suchwow data for the purpose of importing into new model definitions + +import pickle + +from suchwow.models import Post, Moderator, Profile, Ban, AuditEvent +from suchwow import wownero + +wallet = wownero.Wallet() +if not wallet.connected: + print('Wallet not connected') + exit() + +all_posts = Post.select() +all_mods = Moderator.select() +all_profiles = Profile.select() +all_bans = Ban.select() +all_audits = AuditEvent.select() +all_data = { + 'posts': list(), + 'moderators': list(), + 'profiles': list(), + 'bans': list(), + 'auditevents': list() +} + + +for post in all_posts: + all_data['posts'].append({ + 'id': post.id, + 'title': post.title, + 'text': post.text, + 'submitter': post.submitter, + 'image_name': post.image_name, + 'readonly': post.readonly, + 'hidden': post.hidden, + 'account_index': post.account_index, + 'address_index': post.address_index, + 'timestamp': post.timestamp, + 'reddit_url': post.reddit_url, + 'to_reddit': post.to_reddit, + 'to_discord': post.to_discord, + 'approved': post.approved, + 'txes': wallet.transfers(post.account_index) + }) + +for mod in all_mods: + all_data['moderators'].append(mod.username) + +for profile in all_profiles: + all_data['profiles'].append({ + 'username': profile.username, + 'address': profile.address + }) + +for ban in all_bans: + all_data['bans'].append({ + 'username': ban.user.username, + 'reason': ban.reason, + 'timestamp': ban.timestamp + }) + +for event in all_audits: + all_data['auditevents'].append({ + 'username': event.user.username, + 'timestamp': event.timestamp, + 'action': event.action + }) + +with open('data/migrate_data.pkl', 'wb') as f: + f.write(pickle.dumps(all_data)) \ No newline at end of file diff --git a/import.py b/import.py new file mode 100755 index 0000000..8d6242d --- /dev/null +++ b/import.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# import pickled suchwow data for the purpose of importing into new model definitions + +import pickle +from datetime import datetime + +from suchwow._models import User, Post, AuditEvent, TipSent, TipReceived, Vote +from suchwow import wownero + +wallet = wownero.Wallet() +all_data = dict() + +if not wallet.connected: + print('Wallet not running and connected') + exit() + +with open('data/migrate_data.pkl', 'rb') as f: + all_data = pickle.load(f) + +# first import users from old profiles +for user in all_data['profiles']: + if not User.select().where(User.username == user['username']).first(): + u = User( + username=user['username'], + address=user['address'], + ) + u.save() + print(f'Added user {u.username}') + +for post in all_data['posts']: + if not Post.select().where(Post.id == post['id']).first(): + user = User.get(username=post['submitter']) + account_idx = 0 + address_idx, address = wallet.new_address(account_idx) + print(f'Saving post {post["id"]} for user {user.username} (account {account_idx}, address_idx {address_idx}, {address}') + Post.create( + id=post['id'], + title=post['title'], + text=post['text'], + user=user, + image_name=post['image_name'], + account_index=account_idx, + address_index=address_idx, + address=address, + timestamp=post['timestamp'], + approved=post['approved'] + ) + + if 'in' in post['txes']: + p = Post.get(post['id']) + for tx in post['txes']['in']: + if not TipReceived.select().where(TipReceived.txid == tx['txid']).first(): + TipReceived.create( + post=p, + timestamp=datetime.utcfromtimestamp(tx['timestamp']), + txid=tx['txid'], + amount=sum(tx['amounts']), + fee=tx['fee'] + ) + print(f'Saving received tip txid {tx["txid"]}') + + if 'out' in post['txes']: + p = Post.get(post['id']) + for tx in post['txes']['out']: + if not TipSent.select().where(TipSent.txid == tx['txid']).first(): + TipSent.create( + user=p.user, + txid=tx['txid'], + timestamp=datetime.utcfromtimestamp(tx['timestamp']), + amount=tx['amount'], + fee=tx['fee'] + ) + print(f'Saving sent tip txid {tx["txid"]}') + +for mod in all_data['moderators']: + u = User.get(User.username == mod) + if not u.moderator: + u.moderator = True + u.save() + print(f'Updated {u.username} as moderator') + +for ban in all_data['bans']: + u = User.get(User.username == ban['username']) + if not u.banned: + u.banned = True + u.ban_reason = ban['reason'] + u.ban_timestamp = ban['timestamp'] + u.save() + print(f'Banned {u.username} ({u.ban_reason})') + +for event in all_data['auditevents']: + if not AuditEvent.select().where(AuditEvent.timestamp == event['timestamp']): + u = User.get(User.username == event['username']) + AuditEvent.create( + user=u, + action=event['action'], + timestamp=event['timestamp'] + ) + print(f'Saved audit event ({u.username} -> {event["action"]} @ {event["timestamp"]}') \ No newline at end of file diff --git a/suchwow/_models.py b/suchwow/_models.py new file mode 100644 index 0000000..54731c7 --- /dev/null +++ b/suchwow/_models.py @@ -0,0 +1,159 @@ +from random import choice +from os import path +from datetime import datetime + +from peewee import * +from PIL import Image + +from suchwow import wownero +from suchwow import config + + +db = SqliteDatabase(f"{config.DATA_FOLDER}/suchwow_db.sqlite") + +ban_reasons = [ + 'you smell bad', + 'didnt pass the vibe check, homie', + 'your memes are bad and you should feel bad', + 'i just dont like you' +] + +def get_ban_reason(): + return choice(ban_reasons) + + +class User(Model): + id = AutoField() + username = CharField() + address = CharField(null=True) + moderator = BooleanField(default=False) + banned = BooleanField(default=False) + ban_reason = TextField(null=True) + ban_timestamp = DateField(null=True) + login_timestamp = DateTimeField(null=True) + + def __repr__(self): + return self.username + + class Meta: + database = db + + +class Post(Model): + id = AutoField() + title = CharField() + text = CharField(null=True) + user = ForeignKeyField(User) + image_name = CharField() + account_index = IntegerField() + address_index = IntegerField(unique=True) + address = CharField(unique=True) + timestamp = DateTimeField(default=datetime.now) + approved = BooleanField(default=False) + approved_by = ForeignKeyField(User, null=True) + + def get_image_path(self, thumbnail=False): + save_path_base = path.join(config.DATA_FOLDER, "uploads") + if thumbnail: + save_path = path.join(save_path_base, self.get_thumbnail_name()) + else: + save_path = path.join(save_path_base, self.image_name) + return save_path + + def save_thumbnail(self): + try: + image = Image.open(self.get_image_path()) + image.thumbnail((200,200), Image.ANTIALIAS) + image.save(self.get_image_path(True), format=image.format, quality=90) + image.close() + return True + except: + return False + + def get_thumbnail_name(self): + s = path.splitext(self.image_name) + return s[0] + '.thumbnail' + s[1] + + def get_received_wow(self): + try: + w = wownero.Wallet() + it = w.incoming_transfers(self.account_index) + if 'transfers' in it: + amounts = [amt['amount'] for amt in it['transfers'] if 'transfers' in it] + return wownero.as_wownero(wownero.from_atomic(sum(amounts))) + else: + return 0 + except: + return '?' + + def hours_elapsed(self): + now = datetime.utcnow() + diff = now - self.timestamp + return diff.total_seconds() / 60 / 60 + + def show(self): + return { + 'id': self.id, + 'title': self.title, + 'text': self.text, + 'user': self.user.username, + 'image_name': self.image_name, + 'image_path': self.get_image_path(), + 'thumbnail_name': self.get_thumbnail_name(), + 'thumbnail_path': self.get_image_path(True), + 'account_index': self.account_index, + 'address_index': self.address_index, + 'address': self.address, + 'timestamp': self.timestamp, + 'approved': self.approved, + 'approved_by': self.approved_by, + 'received_wow': self.get_received_wow(), + 'hours_elapsed': self.hours_elapsed() + } + + class Meta: + database = db + + +class Vote(Model): + id = AutoField() + post = ForeignKeyField(Post) + upvote = BooleanField() + timestamp = DateTimeField(default=datetime.now) + + class Meta: + database = db + + +class AuditEvent(Model): + id = AutoField() + user = ForeignKeyField(User) + timestamp = DateTimeField(default=datetime.now) + action = CharField() + + class Meta: + database = db + + +class TipReceived(Model): + id = AutoField() + post = ForeignKeyField(Post) + txid = CharField(unique=True) + timestamp = DateTimeField() + amount = IntegerField() + fee = IntegerField() + + class Meta: + database = db + + +class TipSent(Model): + id = AutoField() + user = ForeignKeyField(User) + txid = CharField(unique=True) + timestamp = DateTimeField() + amount = IntegerField() + fee = IntegerField() + + class Meta: + database = db diff --git a/suchwow/cli.py b/suchwow/cli.py index 2ea280d..955b28f 100644 --- a/suchwow/cli.py +++ b/suchwow/cli.py @@ -3,7 +3,7 @@ from os import makedirs import click from flask import Blueprint, url_for, current_app -from suchwow.models import Post, Profile, Comment, Notification, db, Moderator, Ban, AuditEvent +from suchwow._models import db, User, Post, AuditEvent, TipSent, TipReceived, Vote from suchwow.utils.helpers import get_latest_tipped_posts from suchwow.utils.helpers import get_top_posters, get_top_posts from suchwow.reddit import make_post @@ -20,7 +20,7 @@ def init(): makedirs(f"{config.DATA_FOLDER}/{i}", exist_ok=True) # init db - db.create_tables([Post, Profile, Comment, Notification, Moderator, Ban, AuditEvent]) + db.create_tables([User, Post, AuditEvent, TipSent, TipReceived, Vote]) @bp.cli.command("post_reddit") diff --git a/suchwow/wownero.py b/suchwow/wownero.py index d8eb95b..d6c7e3b 100644 --- a/suchwow/wownero.py +++ b/suchwow/wownero.py @@ -88,7 +88,6 @@ class Wallet(object): def new_address(self, account, label=None): data = {'account_index': account, 'label': label} _address = self.make_wallet_rpc('create_address', data) - self.store() return (_address['address_index'], _address['address']) def transfers(self, account, address_indices=[]):