trying to get better state managament

master
lza_menace 4 weeks ago
parent 6a3584224d
commit 3b37020970

@ -45,7 +45,16 @@ class Scrape {
} }
async scrape() { async scrape() {
const pageKey = this.getpageKey() || null const pageKey = this.getpageKey()
if (pageKey === '') {
console.log('no page key')
return
}
db.get('SELECT * FROM events WHERE contract = ? COLLATE NOCASE AND page_key = ? COLLATE NOCASE LIMIT 1', [this.contractAddress, pageKey], async (err, row) => {
if (row) {
console.log('page key exists, skipping');
return
} else {
console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`) console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`)
const response = await alchemy.nft.getNftSales({ const response = await alchemy.nft.getNftSales({
fromBlock: this.startBlock, fromBlock: this.startBlock,
@ -54,7 +63,12 @@ class Scrape {
order: 'asc', order: 'asc',
pageKey: pageKey pageKey: pageKey
}); });
console.log
if (response.pageKey) {
fs.writeFileSync(this.lastFile, response.pageKey) fs.writeFileSync(this.lastFile, response.pageKey)
}
response.nftSales.map(async (sale) => { response.nftSales.map(async (sale) => {
const rowExists = await new Promise((resolve) => { const rowExists = await new Promise((resolve) => {
db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => { db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => {
@ -66,7 +80,7 @@ class Scrape {
try { try {
db.run(` db.run(`
INSERT INTO events VALUES ( INSERT INTO events VALUES (
"${sale.contractAddress}", "${this.contractAddress}",
"${sale.buyerAddress}", "${sale.buyerAddress}",
"${sale.sellerAddress}", "${sale.sellerAddress}",
"${sale.taker}", "${sale.taker}",
@ -80,6 +94,7 @@ class Scrape {
"${sale.logIndex}", "${sale.logIndex}",
"${sale.bundleIndex}", "${sale.bundleIndex}",
"${sale.marketplace}", "${sale.marketplace}",
"${pageKey}",
0, 0 0, 0
)`); )`);
console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`) console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`)
@ -89,6 +104,9 @@ class Scrape {
} }
} }
}); });
}
});
await sleep(1); await sleep(1);
@ -123,6 +141,7 @@ class Scrape {
log_index number, log_index number,
bundle_index number, bundle_index number,
marketplace text, marketplace text,
page_key text,
discord_sent number, discord_sent number,
twitter_sent number, twitter_sent number,
UNIQUE(tx_hash, log_index, bundle_index) UNIQUE(tx_hash, log_index, bundle_index)

Loading…
Cancel
Save