You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
439 lines
16 KiB
JavaScript
439 lines
16 KiB
JavaScript
const { BigNumber, ethers } = require('ethers');
|
|
const fs = require('fs');
|
|
const { Database } = require('sqlite3');
|
|
const { postDiscord } = require('./poster');
|
|
|
|
|
|
if (fs.existsSync('.env.local')) {
|
|
require('dotenv').config({path: '.env.local'});
|
|
} else {
|
|
console.warn('[!] No .env.local found, quitting.');
|
|
process.exit();
|
|
}
|
|
|
|
const CHUNK_SIZE = Number(process.env.CHUNK_SIZE);
|
|
const ALL_CONTRACTS = require('../data/contracts');
|
|
const ERC721_ABI = require('../data/erc721');
|
|
const ERC1155_ABI = require('../data/erc1155');
|
|
const MARKETPLACE_ABI = require('../data/marketplace');
|
|
const SEAPORT_ABI = require('../data/seaport');
|
|
const WYVERN_ABI = require('../data/wyvern');
|
|
const LOOKSRARE_ABI = require('../data/looksrare');
|
|
const WETH_ADDRESS = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2';
|
|
const TRANSFER_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef';
|
|
const TRANSFER_SINGLE_TOPIC = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62';
|
|
const LOOKSRARE_SALE_TOPIC = '0x95fb6205e23ff6bda16a2d1dba56b9ad7c783f67c96fa149785052f47696f2be';
|
|
const SEAPORT_SALE_TOPIC = '0x9d9af8e38d66c62e2c12f0225249fd9d721c54b83f48d9352c97c6cacdcb6f31';
|
|
const WYVERN_SALE_TOPIC = '0xc4109843e0b7d514e4c093114b863f8e7d8d9a458c372cd51bfe526b588006c9'
|
|
const X2Y2_SALE_TOPIC = '0x3cbb63f144840e5b1b0a38a7c19211d2e89de4d7c5faf8b2d3c1776c302d1d33';
|
|
const seaportInterface = new ethers.utils.Interface(SEAPORT_ABI);
|
|
const looksrareInterface = new ethers.utils.Interface(LOOKSRARE_ABI);
|
|
const wyvernInterface = new ethers.utils.Interface(WYVERN_ABI);
|
|
const provider = new ethers.providers.WebSocketProvider(process.env.GETH_NODE);
|
|
const db = new Database('./storage/sqlite.db');
|
|
|
|
|
|
class Collection {
|
|
constructor (contractName) {
|
|
if (!(contractName in ALL_CONTRACTS)) {
|
|
console.warn(`[!] That contract name does not exist in data/contracts.json`);
|
|
process.exit();
|
|
}
|
|
const data = ALL_CONTRACTS[contractName];
|
|
this.contractName = contractName;
|
|
this.contractAddress = data['contract_address'];
|
|
this.erc1155 = data['erc1155'];
|
|
this.startBlock = data['start_block'];
|
|
if (this.erc1155) {
|
|
this.abi = ERC1155_ABI;
|
|
this.transferEvent = 'TransferSingle';
|
|
} else {
|
|
this.abi = ERC721_ABI;
|
|
this.transferEvent = 'Transfer';
|
|
}
|
|
this.interface = new ethers.utils.Interface(this.abi);
|
|
}
|
|
}
|
|
|
|
class Scrape extends Collection {
|
|
|
|
provider = this.getWeb3Provider();
|
|
|
|
constructor (contractName) {
|
|
super(contractName);
|
|
this.contract = new ethers.Contract(this.contractAddress, this.abi, this.provider);
|
|
this.lastFile = `./storage/lastBlock.${this.contractName}.txt`;
|
|
createDatabaseIfNeeded();
|
|
}
|
|
|
|
// ethereum chain provider - geth, infura, alchemy, etc
|
|
getWeb3Provider() {
|
|
return provider;
|
|
}
|
|
|
|
// continuous scanning loop
|
|
async scrape() {
|
|
let latestEthBlock = await this.provider.getBlockNumber();
|
|
let lastScrapedBlock = this.getLastBlock();
|
|
while (true) {
|
|
const lastRequested = lastScrapedBlock;
|
|
|
|
await this.filterTransfers(lastScrapedBlock).then(async ev => {
|
|
// capture transfer events with returned array of Transfers
|
|
try {
|
|
await this.getTransferEvents(ev);
|
|
} catch(err) {
|
|
console.log(ev)
|
|
throw new Error(err);
|
|
}
|
|
// filter down unique transaction hashes
|
|
ev.map(tx => tx.transactionHash).filter((tx, i, a) => a.indexOf(tx) === i).map(async txHash => {
|
|
// capture sales events for each
|
|
try {
|
|
await this.getSalesEvents(txHash);
|
|
} catch(err) {
|
|
console.log(txHash)
|
|
throw new Error(err);
|
|
}
|
|
});
|
|
});
|
|
|
|
if (lastRequested === lastScrapedBlock) {
|
|
lastScrapedBlock += CHUNK_SIZE;
|
|
this.writeLastBlock(lastScrapedBlock);
|
|
if (lastScrapedBlock > latestEthBlock) lastScrapedBlock = latestEthBlock;
|
|
}
|
|
|
|
while (lastScrapedBlock >= latestEthBlock) {
|
|
latestEthBlock = await this.provider.getBlockNumber();
|
|
console.log(`[ ${(new Date()).toISOString()} ][ ${this.contractName} ] [ waiting ]\n`)
|
|
await sleep(120);
|
|
}
|
|
}
|
|
}
|
|
|
|
// query historical logs
|
|
async filterTransfers(startBlock) {
|
|
let transfers;
|
|
console.log(`[ ${(new Date()).toISOString()} ][ ${this.contractName} ][ scraping ] blocks ${startBlock} - ${startBlock + CHUNK_SIZE}\n`);
|
|
if (this.erc1155) {
|
|
transfers = this.contract.filters.TransferSingle(null, null);
|
|
} else {
|
|
transfers = this.contract.filters.Transfer(null, null);
|
|
}
|
|
let res = await this.contract.queryFilter(transfers, startBlock, startBlock + CHUNK_SIZE)
|
|
return res;
|
|
}
|
|
|
|
// get transfer events from a batch from filtering
|
|
async getTransferEvents(txEvents) {
|
|
let platform = 'contract';
|
|
txEvents.forEach(async tx => {
|
|
let tokenId;
|
|
if (this.erc1155) {
|
|
tokenId = tx.args.id.toString();
|
|
} else {
|
|
tokenId = tx.args.tokenId.toString();
|
|
}
|
|
const fromAddress = tx.args.from.toString().toLowerCase();
|
|
const toAddress = tx.args.to.toString().toLowerCase();
|
|
const timestamp = await this.getBlockTimestamp(tx.blockNumber);
|
|
let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ transfer ] #${tokenId}: ${fromAddress} => ${toAddress} in tx ${tx.transactionHash}:${tx.logIndex}\n`;
|
|
console.log(msg);
|
|
const q = {
|
|
txHash: tx.transactionHash,
|
|
logIndex: tx.logIndex,
|
|
contractName: this.contractName,
|
|
contractAddress: this.contractAddress,
|
|
eventName: "transfer",
|
|
eventSource: "contract",
|
|
sourceOwner: fromAddress,
|
|
targetOwner: toAddress,
|
|
tokenId: tokenId,
|
|
amount: 0,
|
|
txDate: timestamp
|
|
}
|
|
writeToDatabase(q)
|
|
.then((res) => this.writeLastBlock(tx.blockNumber))
|
|
.catch((err) => console.log(`Error writing to database: ${err}`));
|
|
});
|
|
}
|
|
|
|
// get sales events from a given transaction
|
|
async getSalesEvents(txHash) {
|
|
try {
|
|
const receipt = await this.provider.getTransactionReceipt(txHash);
|
|
const timestamp = await this.getBlockTimestamp(receipt.blockNumber);
|
|
// Evaluate each log entry and determine if it's a sale for our contract and use custom logic for each exchange to parse values
|
|
receipt.logs.map(async log => {
|
|
let logIndex = log.logIndex;
|
|
let sale = false;
|
|
let platform;
|
|
let fromAddress;
|
|
let toAddress;
|
|
let amountWei;
|
|
let amountEther;
|
|
let tokenId;
|
|
if (log.topics[0].toLowerCase() === SEAPORT_SALE_TOPIC.toLowerCase()) {
|
|
// Handle Opensea/Seaport sales
|
|
const logDescription = seaportInterface.parseLog(log)
|
|
const matchingOffers = logDescription.args.offer.filter(
|
|
o => o.token.toLowerCase() == this.contractAddress.toLowerCase()
|
|
);
|
|
if (matchingOffers.length === 0) return;
|
|
sale = true;
|
|
platform = 'opensea';
|
|
fromAddress = logDescription.args.offerer.toLowerCase();
|
|
toAddress = logDescription.args.recipient.toLowerCase();
|
|
tokenId = logDescription.args.offer.map(o => o.identifier.toString());
|
|
let amounts = logDescription.args.consideration.map(c => BigInt(c.amount));
|
|
// add weth
|
|
const wethOffers = matchingOffers.map(o => o.token.toLowerCase() === WETH_ADDRESS.toLowerCase() && o.amount > 0 ? BigInt(o.amount) : BigInt(0));
|
|
if (wethOffers.length > 0 && wethOffers[0] != BigInt(0)) {
|
|
amounts = wethOffers
|
|
}
|
|
amountWei = amounts.reduce((previous,current) => previous + current, BigInt(0));
|
|
} else if (log.topics[0].toLowerCase() === WYVERN_SALE_TOPIC.toLowerCase()) {
|
|
// Handle Opensea/Wyvern sales
|
|
const logDescription = wyvernInterface.parseLog(log);
|
|
sale = true;
|
|
platform = 'opensea';
|
|
if (this.erc1155) {
|
|
const txLog = receipt.logs.map(l => l).filter(_l =>
|
|
(_l.topics[0].toLowerCase() == TRANSFER_SINGLE_TOPIC.toLowerCase())
|
|
).map(t => this.interface.parseLog(t))[0].args;
|
|
fromAddress = txLog.from.toLowerCase();
|
|
toAddress = txLog.to.toLowerCase();
|
|
tokenId = BigNumber.from(txLog.id).toString();
|
|
} else {
|
|
const txLog = receipt.logs.map(l => l).filter(_l =>
|
|
(_l.topics[0].toLowerCase() == TRANSFER_TOPIC.toLowerCase())
|
|
).map(t => this.interface.parseLog(t))[0].args;
|
|
fromAddress = txLog.from.toLowerCase();
|
|
toAddress = txLog.to.toLowerCase();
|
|
tokenId = BigNumber.from(txLog.tokenId).toString();
|
|
}
|
|
amountWei = BigInt(logDescription.args.price);
|
|
} else if (log.topics[0].toLowerCase() === LOOKSRARE_SALE_TOPIC.toLowerCase()) {
|
|
// Handle LooksRare sales
|
|
const logDescription = looksrareInterface.parseLog(log);
|
|
if (logDescription.args.collection.toLowerCase() != this.contractAddress.toLowerCase()) return;
|
|
sale = true;
|
|
platform = 'looksrare';
|
|
fromAddress = logDescription.args.maker.toLowerCase();
|
|
toAddress = receipt.from.toLowerCase();
|
|
tokenId = logDescription.args.tokenId.toString();
|
|
amountWei = logDescription.args.price.toString();
|
|
} else if (log.topics[0].toLowerCase() === X2Y2_SALE_TOPIC.toLowerCase()) {
|
|
// Handle x2y2 sales
|
|
const data = log.data.substring(2);
|
|
const dataSlices = data.match(/.{1,64}/g);
|
|
sale = true;
|
|
platform = 'x2y2';
|
|
fromAddress = BigNumber.from(`0x${dataSlices[0]}`)._hex.toString().toLowerCase();
|
|
toAddress = BigNumber.from(`0x${dataSlices[1]}`)._hex.toString().toLowerCase();
|
|
tokenId = BigInt(`0x${dataSlices[18]}`).toString();
|
|
amountWei = BigInt(`0x${dataSlices[12]}`);
|
|
if (amountWei === BigInt(0)) {
|
|
amountWei = BigInt(`0x${dataSlices[26]}`);
|
|
}
|
|
}
|
|
if (sale) {
|
|
amountEther = ethers.utils.formatEther(amountWei);
|
|
let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ sale ] #${tokenId}: ${fromAddress} => ${toAddress} for ${amountEther}Ξ (${platform}) in tx ${txHash}:${logIndex}\n`;
|
|
console.log(msg);
|
|
const q = {
|
|
txHash: txHash,
|
|
logIndex: logIndex,
|
|
contractName: this.contractName,
|
|
contractAddress: this.contractAddress,
|
|
eventName: 'sale',
|
|
eventSource: platform,
|
|
sourceOwner: fromAddress,
|
|
targetOwner: toAddress,
|
|
tokenId: tokenId,
|
|
amount: amountWei,
|
|
txDate: timestamp
|
|
}
|
|
writeToDatabase(q)
|
|
.then((res) => this.writeLastBlock(log.blockNumber))
|
|
.catch((err) => console.log(`Error writing to database: ${err}`));
|
|
let notifSent = await checkUnsentNotif(txHash, logIndex);
|
|
if (process.env.DISCORD_ACTIVE == 1 && (notifSent || process.env.FORCE == 1)) {
|
|
postDiscord(q)
|
|
.then(async res => {
|
|
await markSent(txHash, logIndex);
|
|
console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}\n`)
|
|
})
|
|
.catch((err) => console.log(`Error posting to Discord: ${err}`));
|
|
}
|
|
}
|
|
});
|
|
} catch(err) {
|
|
console.log(err);
|
|
}
|
|
}
|
|
|
|
/* Helpers */
|
|
|
|
// get stored block index to start scraping from
|
|
getLastBlock() {
|
|
let last = 0;
|
|
if (fs.existsSync(this.lastFile)) {
|
|
// read block stored in lastBlock file
|
|
last = parseInt(fs.readFileSync(this.lastFile).toString(), 10);
|
|
} else {
|
|
// write starting block if lastBlock file doesn't exist
|
|
fs.writeFileSync(this.lastFile, this.startBlock.toString());
|
|
};
|
|
// contract creation
|
|
if (Number.isNaN(last) || last < this.startBlock) {
|
|
last = this.startBlock;
|
|
};
|
|
return last;
|
|
}
|
|
|
|
// write last block to local filesystem
|
|
writeLastBlock(blockNumber) {
|
|
fs.writeFileSync(this.lastFile, blockNumber.toString());
|
|
}
|
|
|
|
// return date object of a given block
|
|
async getBlockTimestamp(blockNumber) {
|
|
const block = await this.provider.getBlock(blockNumber);
|
|
const d = new Date(block.timestamp * 1000);
|
|
return d;
|
|
}
|
|
|
|
}
|
|
|
|
async function sleep(sec) {
|
|
return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000));
|
|
}
|
|
|
|
async function createDatabaseIfNeeded() {
|
|
const tableExists = await new Promise((resolve) => {
|
|
db.get('SELECT name FROM sqlite_master WHERE type="table" AND name="events"', [], (err, row) => {
|
|
if (err) {
|
|
resolve(false);
|
|
}
|
|
resolve(row !== undefined);
|
|
});
|
|
});
|
|
if (!tableExists) {
|
|
db.serialize(() => {
|
|
db.run(
|
|
`CREATE TABLE events (
|
|
contract text, event_type text, from_wallet text, to_wallet text,
|
|
token_id number, amount number, tx_date text, tx text,
|
|
log_index number, platform text, discord_sent number, twitter_sent number,
|
|
UNIQUE(tx, log_index)
|
|
);`,
|
|
);
|
|
db.run('CREATE INDEX idx_type_date ON events(event_type, tx_date);');
|
|
db.run('CREATE INDEX idx_date ON events(tx_date);');
|
|
db.run('CREATE INDEX idx_amount ON events(amount);');
|
|
db.run('CREATE INDEX idx_platform ON events(platform);');
|
|
db.run('CREATE INDEX idx_contract ON events(contract);');
|
|
db.run('CREATE INDEX idx_tx ON events(tx);');
|
|
});
|
|
}
|
|
}
|
|
|
|
async function checkRowExists(txHash, logIndex) {
|
|
const rowExists = await new Promise((resolve) => {
|
|
db.get('SELECT * FROM events WHERE tx = ? AND log_index = ?', [txHash, logIndex], (err, row) => {
|
|
if (err) {
|
|
resolve(false);
|
|
}
|
|
resolve(row !== undefined);
|
|
});
|
|
});
|
|
return rowExists;
|
|
}
|
|
|
|
async function checkUnsentNotif(txHash, logIndex) {
|
|
const rowExists = await new Promise((resolve) => {
|
|
db.get('SELECT * FROM events WHERE tx = ? AND log_index = ? AND discord_sent != 1', [txHash, logIndex], (err, row) => {
|
|
if (err) {
|
|
resolve(false);
|
|
}
|
|
resolve(row !== undefined);
|
|
});
|
|
});
|
|
return rowExists;
|
|
}
|
|
|
|
async function markSent(txHash, logIndex) {
|
|
try {
|
|
const stmt = db.prepare('UPDATE events SET discord_sent = 1 WHERE tx = ? AND log_index = ?');
|
|
stmt.run(txHash, logIndex);
|
|
stmt.finalize();
|
|
} catch(err) {
|
|
console.log(`Error writing to database: ${err}`)
|
|
}
|
|
}
|
|
|
|
async function writeToDatabase(_q) {
|
|
// txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate
|
|
const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress);
|
|
if (!rowExists) {
|
|
let stmt;
|
|
try {
|
|
stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)');
|
|
stmt.run(
|
|
_q.contractAddress,
|
|
_q.eventName,
|
|
_q.sourceOwner,
|
|
_q.targetOwner,
|
|
_q.tokenId,
|
|
_q.amount.toString(),
|
|
_q.txDate.toISOString(),
|
|
_q.txHash,
|
|
_q.logIndex,
|
|
_q.eventSource,
|
|
0,
|
|
0
|
|
);
|
|
stmt.finalize();
|
|
return true;
|
|
} catch(err) {
|
|
console.log(`Error when writing to database: ${err}`);
|
|
console.log(`Query: ${stmt}`);
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Sample events for testing functionality and detecting sales
|
|
// let c = new Scrape('non-fungible-soup');
|
|
// c.getSalesEvents('0x2f8961209daca23288c499449aa936b54eec5c25720b9d7499a8ee5bde7fcdc7')
|
|
// c.getSalesEvents('0xb20853f22b367ee139fd800206bf1cba0c36f1a1dd739630f99cc6ffd0471edc')
|
|
// c.getSalesEvents('0x71e5135a543e17cc91992a2229ae5811461c96b84d5e2560ac8db1dd99bb17e3')
|
|
// c.getSalesEvents('0x5dc68e0bd60fa671e7b6702002e4ce374de6a5dd49fcda00fdb45e26771bcbd9')
|
|
// c.getSalesEvents('0x975d10cdd873ee5bb29e746c2f1f3b776078cace9c04ce419cb66949239288b5')
|
|
// c.getSalesEvents('0x8d45ed8168a740f8b182ec0dbad1c37d6c6dbd8aa865be408d865ca01fb0fa94')
|
|
// c.getSalesEvents('0x27ab6f12604bf17a9e7c93bf1a7cc466d7dfd922565d267eac10879b59d5d0b5')
|
|
// c.getSalesEvents('0x511bc5cda2b7145511c7b57e29cecf1f15a5a650670f09e91e69fc24824effd9')
|
|
// c.getSalesEvents('0x04746b6ba1269906db8e0932263b86a6fc35a30a31cf73d2b7db078f6f4ed442')
|
|
// c.getSalesEvents('0x24d6523c5048b2df3e7f8b24d63a6644e4c0ed33cfae6396190e3ded5fc79321')
|
|
// c.getSalesEvents('0xe56dc64c44a3cbfe3a1e68f8669a65f17ebe48d64e944673122a565b7c641d1e')
|
|
// c.getSalesEvents('0xe567d00bb0c16928d5d8c258de8dd928e93209b40f7c958bc485d2a6c549b8a9')
|
|
// return
|
|
|
|
if (process.env.SCRAPE) {
|
|
let c = new Scrape(process.env.SCRAPE)
|
|
c.scrape()
|
|
} else {
|
|
for(const key in ALL_CONTRACTS) {
|
|
if (process.env.ONLY && process.env.ONLY != key) continue
|
|
const c = new Scrape(key);
|
|
if (process.env.TX) {
|
|
c.getSalesEvents(process.env.TX);
|
|
continue;
|
|
}
|
|
c.scrape();
|
|
}
|
|
}
|