setup backups/restores, fix discord posting (enforce 1 time)

pull/1/head
lza_menace 2 years ago
parent b3736d627a
commit b89f6db5d2

1
.gitignore vendored

@ -1,5 +1,6 @@
.env.local .env.local
node_modules node_modules
storage/*.json
storage/*.txt storage/*.txt
storage/*.db storage/*.db
data/contracts.json data/contracts.json

@ -0,0 +1,47 @@
const Database = require('better-sqlite3');
const fs = require('fs');
if (fs.existsSync('.env.local')) {
require('dotenv').config({path: '.env.local'});
} else {
console.warn('[!] No .env.local found, quitting.');
process.exit();
}
const db = new Database('./storage/sqlite.db');
const backupPath = './storage/backup.json';
let l = process.argv.filter((val, idx, arr) => idx == 2)[0]
if (l == 'backup') {
console.log(`performing backup of SQLite data to ${backupPath}`);
const results = [];
const stmt = db.prepare(`SELECT * FROM events ORDER BY tx_date DESC`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
fs.writeFileSync(backupPath, JSON.stringify(results));
console.log(`[+] Wrote ${results.length} records to ${backupPath}`);
} else if (l == 'restore') {
console.log(`restoring backup of SQLite data from ${backupPath}`);
const backupData = require('../storage/backup.json');
console.log(`deleting old data first`);
const deleteEvents = db.prepare(`DELETE FROM events where event_type != 'yolo'`);
deleteEvents.run();
console.log(`inserting new data from backup file`)
const insertEvent = db.prepare('INSERT INTO events (contract, event_type, from_wallet, to_wallet, token_id, amount, tx_date, tx, log_index, platform, discord_sent, twitter_sent) VALUES (@contract, @event_type, @from_wallet, @to_wallet, @token_id, @amount, @tx_date, @tx, @log_index, @platform, @discord_sent, @twitter_sent)');
const insertEvents = db.transaction((events) => {
for (let ev of events) {
if (!ev.discord_sent) {
ev.discord_sent = 0;
}
if (!ev.twitter_sent) {
ev.twitter_sent = 0;
}
insertEvent.run(ev)
};
});
insertEvents(backupData);
} else {
console.log(`[!] Invalid arguments provided, quitting!`)
process.exit();
}

@ -41,7 +41,7 @@ class Collection {
} }
const data = ALL_CONTRACTS[contractName]; const data = ALL_CONTRACTS[contractName];
this.contractName = contractName; this.contractName = contractName;
this.contractAddress = data['contract_address'].toLowerCase(); this.contractAddress = data['contract_address'];
this.erc1155 = data['erc1155']; this.erc1155 = data['erc1155'];
this.startBlock = data['start_block']; this.startBlock = data['start_block'];
if (this.erc1155) { if (this.erc1155) {
@ -258,9 +258,14 @@ class Scrape extends Collection {
writeToDatabase(q) writeToDatabase(q)
.then((res) => this.writeLastBlock(log.blockNumber)) .then((res) => this.writeLastBlock(log.blockNumber))
.catch((err) => console.log(`Error writing to database: ${err}`)); .catch((err) => console.log(`Error writing to database: ${err}`));
postDiscord(q) if (checkUnsentNotif(txHash, logIndex)) {
.then((res) => console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`)) postDiscord(q)
.catch((err) => console.log(`Error posting to Discord: ${err}`)); .then(async res => {
await markSent(txHash, logIndex);
console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`)
})
.catch((err) => console.log(`Error posting to Discord: ${err}`));
}
} }
}); });
} catch(err) { } catch(err) {
@ -320,7 +325,7 @@ async function createDatabaseIfNeeded() {
`CREATE TABLE events ( `CREATE TABLE events (
contract text, event_type text, from_wallet text, to_wallet text, contract text, event_type text, from_wallet text, to_wallet text,
token_id number, amount number, tx_date text, tx text, token_id number, amount number, tx_date text, tx text,
log_index number, platform text, log_index number, platform text, discord_sent number, twitter_sent number,
UNIQUE(tx, log_index) UNIQUE(tx, log_index)
);`, );`,
); );
@ -346,13 +351,35 @@ async function checkRowExists(txHash, logIndex) {
return rowExists; return rowExists;
} }
async function checkUnsentNotif(txHash, logIndex) {
const rowExists = await new Promise((resolve) => {
db.get('SELECT * FROM events WHERE tx = ? AND log_index = ? AND discord_sent != 1', [txHash, logIndex], (err, row) => {
if (err) {
resolve(false);
}
resolve(row !== undefined);
});
});
return rowExists;
}
async function markSent(txHash, logIndex) {
try {
const stmt = db.prepare('UPDATE events SET discord_sent = 1 WHERE tx = ? AND log_index = ?');
stmt.run(txHash, logIndex);
stmt.finalize();
} catch(err) {
console.log(`Error writing to database: ${err}`)
}
}
async function writeToDatabase(_q) { async function writeToDatabase(_q) {
// txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate // txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate
const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress); const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress);
if (!rowExists) { if (!rowExists) {
let stmt; let stmt;
try { try {
stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?)'); stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)');
stmt.run( stmt.run(
_q.contractAddress, _q.contractAddress,
_q.eventName, _q.eventName,
@ -363,7 +390,9 @@ async function writeToDatabase(_q) {
_q.txDate.toISOString(), _q.txDate.toISOString(),
_q.txHash, _q.txHash,
_q.logIndex, _q.logIndex,
_q.eventSource _q.eventSource,
0,
0
); );
stmt.finalize(); stmt.finalize();
return true; return true;
@ -397,6 +426,7 @@ if (process.env.SCRAPE) {
c.scrape() c.scrape()
} else { } else {
for(const key in ALL_CONTRACTS) { for(const key in ALL_CONTRACTS) {
if (process.env.ONLY && process.env.ONLY != key) continue
const c = new Scrape(key); const c = new Scrape(key);
c.scrape(); c.scrape();
} }

Loading…
Cancel
Save