setup backups/restores, fix discord posting (enforce 1 time)

pull/1/head
lza_menace 2 years ago
parent b3736d627a
commit b89f6db5d2

1
.gitignore vendored

@ -1,5 +1,6 @@
.env.local
node_modules
storage/*.json
storage/*.txt
storage/*.db
data/contracts.json

@ -0,0 +1,47 @@
const Database = require('better-sqlite3');
const fs = require('fs');
if (fs.existsSync('.env.local')) {
require('dotenv').config({path: '.env.local'});
} else {
console.warn('[!] No .env.local found, quitting.');
process.exit();
}
const db = new Database('./storage/sqlite.db');
const backupPath = './storage/backup.json';
let l = process.argv.filter((val, idx, arr) => idx == 2)[0]
if (l == 'backup') {
console.log(`performing backup of SQLite data to ${backupPath}`);
const results = [];
const stmt = db.prepare(`SELECT * FROM events ORDER BY tx_date DESC`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
fs.writeFileSync(backupPath, JSON.stringify(results));
console.log(`[+] Wrote ${results.length} records to ${backupPath}`);
} else if (l == 'restore') {
console.log(`restoring backup of SQLite data from ${backupPath}`);
const backupData = require('../storage/backup.json');
console.log(`deleting old data first`);
const deleteEvents = db.prepare(`DELETE FROM events where event_type != 'yolo'`);
deleteEvents.run();
console.log(`inserting new data from backup file`)
const insertEvent = db.prepare('INSERT INTO events (contract, event_type, from_wallet, to_wallet, token_id, amount, tx_date, tx, log_index, platform, discord_sent, twitter_sent) VALUES (@contract, @event_type, @from_wallet, @to_wallet, @token_id, @amount, @tx_date, @tx, @log_index, @platform, @discord_sent, @twitter_sent)');
const insertEvents = db.transaction((events) => {
for (let ev of events) {
if (!ev.discord_sent) {
ev.discord_sent = 0;
}
if (!ev.twitter_sent) {
ev.twitter_sent = 0;
}
insertEvent.run(ev)
};
});
insertEvents(backupData);
} else {
console.log(`[!] Invalid arguments provided, quitting!`)
process.exit();
}

@ -41,7 +41,7 @@ class Collection {
}
const data = ALL_CONTRACTS[contractName];
this.contractName = contractName;
this.contractAddress = data['contract_address'].toLowerCase();
this.contractAddress = data['contract_address'];
this.erc1155 = data['erc1155'];
this.startBlock = data['start_block'];
if (this.erc1155) {
@ -258,10 +258,15 @@ class Scrape extends Collection {
writeToDatabase(q)
.then((res) => this.writeLastBlock(log.blockNumber))
.catch((err) => console.log(`Error writing to database: ${err}`));
if (checkUnsentNotif(txHash, logIndex)) {
postDiscord(q)
.then((res) => console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`))
.then(async res => {
await markSent(txHash, logIndex);
console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}`)
})
.catch((err) => console.log(`Error posting to Discord: ${err}`));
}
}
});
} catch(err) {
console.log(err);
@ -320,7 +325,7 @@ async function createDatabaseIfNeeded() {
`CREATE TABLE events (
contract text, event_type text, from_wallet text, to_wallet text,
token_id number, amount number, tx_date text, tx text,
log_index number, platform text,
log_index number, platform text, discord_sent number, twitter_sent number,
UNIQUE(tx, log_index)
);`,
);
@ -346,13 +351,35 @@ async function checkRowExists(txHash, logIndex) {
return rowExists;
}
async function checkUnsentNotif(txHash, logIndex) {
const rowExists = await new Promise((resolve) => {
db.get('SELECT * FROM events WHERE tx = ? AND log_index = ? AND discord_sent != 1', [txHash, logIndex], (err, row) => {
if (err) {
resolve(false);
}
resolve(row !== undefined);
});
});
return rowExists;
}
async function markSent(txHash, logIndex) {
try {
const stmt = db.prepare('UPDATE events SET discord_sent = 1 WHERE tx = ? AND log_index = ?');
stmt.run(txHash, logIndex);
stmt.finalize();
} catch(err) {
console.log(`Error writing to database: ${err}`)
}
}
async function writeToDatabase(_q) {
// txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate
const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress);
if (!rowExists) {
let stmt;
try {
stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?)');
stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)');
stmt.run(
_q.contractAddress,
_q.eventName,
@ -363,7 +390,9 @@ async function writeToDatabase(_q) {
_q.txDate.toISOString(),
_q.txHash,
_q.logIndex,
_q.eventSource
_q.eventSource,
0,
0
);
stmt.finalize();
return true;
@ -397,6 +426,7 @@ if (process.env.SCRAPE) {
c.scrape()
} else {
for(const key in ALL_CONTRACTS) {
if (process.env.ONLY && process.env.ONLY != key) continue
const c = new Scrape(key);
c.scrape();
}

Loading…
Cancel
Save