From b89f6db5d2c66861d686f2d114b972be0553bc17 Mon Sep 17 00:00:00 2001 From: lza_menace Date: Sun, 31 Jul 2022 08:58:48 -0700 Subject: [PATCH] setup backups/restores, fix discord posting (enforce 1 time) --- .gitignore | 1 + src/dbmgmt.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/scraper.js | 44 +++++++++++++++++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 src/dbmgmt.js diff --git a/.gitignore b/.gitignore index f1df3fd..e0b7409 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .env.local node_modules +storage/*.json storage/*.txt storage/*.db data/contracts.json diff --git a/src/dbmgmt.js b/src/dbmgmt.js new file mode 100644 index 0000000..22f58be --- /dev/null +++ b/src/dbmgmt.js @@ -0,0 +1,47 @@ +const Database = require('better-sqlite3'); +const fs = require('fs'); + +if (fs.existsSync('.env.local')) { + require('dotenv').config({path: '.env.local'}); +} else { + console.warn('[!] No .env.local found, quitting.'); + process.exit(); +} + +const db = new Database('./storage/sqlite.db'); +const backupPath = './storage/backup.json'; + +let l = process.argv.filter((val, idx, arr) => idx == 2)[0] +if (l == 'backup') { + console.log(`performing backup of SQLite data to ${backupPath}`); + const results = []; + const stmt = db.prepare(`SELECT * FROM events ORDER BY tx_date DESC`); + for (const entry of stmt.iterate()) { + results.push(entry); + } + fs.writeFileSync(backupPath, JSON.stringify(results)); + console.log(`[+] Wrote ${results.length} records to ${backupPath}`); +} else if (l == 'restore') { + console.log(`restoring backup of SQLite data from ${backupPath}`); + const backupData = require('../storage/backup.json'); + console.log(`deleting old data first`); + const deleteEvents = db.prepare(`DELETE FROM events where event_type != 'yolo'`); + deleteEvents.run(); + console.log(`inserting new data from backup file`) + const insertEvent = db.prepare('INSERT INTO events (contract, event_type, from_wallet, to_wallet, token_id, amount, tx_date, tx, log_index, platform, discord_sent, twitter_sent) VALUES (@contract, @event_type, @from_wallet, @to_wallet, @token_id, @amount, @tx_date, @tx, @log_index, @platform, @discord_sent, @twitter_sent)'); + const insertEvents = db.transaction((events) => { + for (let ev of events) { + if (!ev.discord_sent) { + ev.discord_sent = 0; + } + if (!ev.twitter_sent) { + ev.twitter_sent = 0; + } + insertEvent.run(ev) + }; + }); + insertEvents(backupData); +} else { + console.log(`[!] Invalid arguments provided, quitting!`) + process.exit(); +} diff --git a/src/scraper.js b/src/scraper.js index bdbe1d6..2c03ea3 100644 --- a/src/scraper.js +++ b/src/scraper.js @@ -41,7 +41,7 @@ class Collection { } const data = ALL_CONTRACTS[contractName]; this.contractName = contractName; - this.contractAddress = data['contract_address'].toLowerCase(); + this.contractAddress = data['contract_address']; this.erc1155 = data['erc1155']; this.startBlock = data['start_block']; if (this.erc1155) { @@ -258,9 +258,14 @@ class Scrape extends Collection { writeToDatabase(q) .then((res) => this.writeLastBlock(log.blockNumber)) .catch((err) => console.log(`Error writing to database: ${err}`)); - postDiscord(q) - .then((res) => console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`)) - .catch((err) => console.log(`Error posting to Discord: ${err}`)); + if (checkUnsentNotif(txHash, logIndex)) { + postDiscord(q) + .then(async res => { + await markSent(txHash, logIndex); + console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`) + }) + .catch((err) => console.log(`Error posting to Discord: ${err}`)); + } } }); } catch(err) { @@ -320,7 +325,7 @@ async function createDatabaseIfNeeded() { `CREATE TABLE events ( contract text, event_type text, from_wallet text, to_wallet text, token_id number, amount number, tx_date text, tx text, - log_index number, platform text, + log_index number, platform text, discord_sent number, twitter_sent number, UNIQUE(tx, log_index) );`, ); @@ -346,13 +351,35 @@ async function checkRowExists(txHash, logIndex) { return rowExists; } +async function checkUnsentNotif(txHash, logIndex) { + const rowExists = await new Promise((resolve) => { + db.get('SELECT * FROM events WHERE tx = ? AND log_index = ? AND discord_sent != 1', [txHash, logIndex], (err, row) => { + if (err) { + resolve(false); + } + resolve(row !== undefined); + }); + }); + return rowExists; +} + +async function markSent(txHash, logIndex) { + try { + const stmt = db.prepare('UPDATE events SET discord_sent = 1 WHERE tx = ? AND log_index = ?'); + stmt.run(txHash, logIndex); + stmt.finalize(); + } catch(err) { + console.log(`Error writing to database: ${err}`) + } +} + async function writeToDatabase(_q) { // txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress); if (!rowExists) { let stmt; try { - stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?)'); + stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)'); stmt.run( _q.contractAddress, _q.eventName, @@ -363,7 +390,9 @@ async function writeToDatabase(_q) { _q.txDate.toISOString(), _q.txHash, _q.logIndex, - _q.eventSource + _q.eventSource, + 0, + 0 ); stmt.finalize(); return true; @@ -397,6 +426,7 @@ if (process.env.SCRAPE) { c.scrape() } else { for(const key in ALL_CONTRACTS) { + if (process.env.ONLY && process.env.ONLY != key) continue const c = new Scrape(key); c.scrape(); }