diff --git a/bun.lockb b/bun.lockb index 177ff5b..fb98424 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index dbc28f6..4a533d1 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "dependencies": { - "alchemy-sdk": "^3.5.0", "better-sqlite3": "^11.7.0", "ethers": "^6.13.4", "express": "^4.21.2", + "moralis": "^2.27.2", "sqlite3": "^5.1.7" }, - "name": "alchemy-nft-scraper", + "name": "nft-sales-scraper", "module": "index.ts", "devDependencies": { "@types/bun": "latest" diff --git a/src/index.js b/src/index.js index e214784..38eef26 100644 --- a/src/index.js +++ b/src/index.js @@ -1,18 +1,15 @@ const ALL_CONTRACTS = require('./contracts'); -const { Alchemy, Network } = require("alchemy-sdk"); +import Moralis from 'moralis'; const { Database } = require('sqlite3'); const fs = require('fs'); const db = new Database('./state/sqlite.db'); const config = { - apiKey: process.env.ALCHEMY_KEY, - network: Network.ETH_MAINNET, + apiKey: process.env.MORALIS_KEY }; -const alchemy = new Alchemy(config); - async function sleep(sec) { return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000)); @@ -34,7 +31,7 @@ class Scrape { this.lastFile = `./state/${this.contractName}.txt`; } - getpageKey() { + getCursor() { if (fs.existsSync(this.lastFile)) { return fs.readFileSync(this.lastFile).toString(); } else { @@ -44,57 +41,56 @@ class Scrape { } async scrape() { - const pageKey = this.getpageKey() - if (pageKey === '') { - console.log('no page key') + const cursor = this.getCursor() + if (cursor === '') { + console.log(`no cursor for ${this.contractName}. skipping`) return } - console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`) - const response = await alchemy.nft.getNftSales({ + console.log(`[+] Scraping ${this.contractName}`); + const response = await Moralis.EvmApi.nft.getNFTTrades({ + chain: '0x1', + marketplace: 'opensea', fromBlock: this.startBlock, - contractAddress: this.contractAddress, + address: this.contractAddress, limit: process.env.LIMIT, - order: 'asc', - pageKey: pageKey + cursor: cursor }); - fs.writeFileSync(this.lastFile, response.pageKey || '') + fs.writeFileSync(this.lastFile, response.json.cursor || '') - response.nftSales.map(async (sale) => { - const rowExists = await new Promise((resolve) => { - db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => { - if (err) { resolve(false); } - resolve(row !== undefined); + response.json.result.map(async (sale) => { + sale.token_ids.map(async (tokenId) => { + const rowExists = await new Promise((resolve) => { + db.get('SELECT * FROM events WHERE tx_hash = ? AND token_id = ?', [sale.transaction_hash, tokenId], (err, row) => { + if (err) { resolve(false); } + resolve(row !== undefined); + }); }); - }); - if (!rowExists) { - try { - db.run(` - INSERT INTO events VALUES ( - "${this.contractAddress}", - "${sale.buyerAddress}", - "${sale.sellerAddress}", - "${sale.taker}", - "${sale.tokenId}", - "${sale.sellerFee.amount}", - "${sale.protocolFee.amount}", - "${sale.royaltyFee.amount}", - "", - "${sale.transactionHash}", - "${sale.blockNumber}", - "${sale.logIndex}", - "${sale.bundleIndex}", - "${sale.marketplace}", - "${pageKey}", - 0, 0 - )`); - console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`) - } catch(err) { - console.log(`Error when writing to database: ${err}`); - return false; + if (!rowExists) { + try { + db.run(` + INSERT INTO events VALUES ( + "${this.contractAddress}", + "${sale.buyer_address}", + "${sale.seller_address}", + "${tokenId}", + "${sale.price}", + "", + "${sale.transaction_hash}", + "${sale.block_number}", + "opensea", + "${cursor}", + 0, 0 + )`); + console.log(` ::: Inserted sale of ${this.contractName} #${tokenId} in block ${sale.block_number} for ${sale.price} wei.`) + } catch(err) { + console.log(`Error when writing to database: ${err}`); + return false; + } } - } + }); + }); await sleep(1); @@ -119,25 +115,21 @@ class Scrape { contract text, buyer text, seller text, - taker text, token_id number, sale_price text, - protocol_fee text, - royalty_fee text, tx_date text, tx_hash text, block_number number, - log_index number, - bundle_index number, marketplace text, - page_key text, + cursor text, discord_sent number, twitter_sent number, - UNIQUE(tx_hash, log_index, bundle_index) + UNIQUE(tx_hash, token_id) );`, ); }); } + await Moralis.start(config); while(true) { for(const contract in ALL_CONTRACTS) { if (process.env.ONLY && process.env.ONLY != contract) continue diff --git a/src/server.js b/src/server.js index 9a33171..6a807d1 100644 --- a/src/server.js +++ b/src/server.js @@ -61,7 +61,7 @@ app.get('/api/latest', (req, res) => { app.get('/api/:contractAddress/data', (req, res) => { const results = []; const stmt = db.prepare(`select - block_number block, + (block_number / 100) * 100 block, sum(sale_price/1000000000000000000.0) volume, avg(sale_price/1000000000000000000.0) average_price, (select avg(sale_price/1000000000000000000.0) from (select * from events