setting up x2y2 interface instead of data parsing

10 changed files with 162 additions and 1452 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,5 @@
 .env.local
 node_modules
-storage/*.json
 storage/*.txt
 storage/*.db
 data/contracts.json
--- a/data/blur.json
+++ b/data/blur.json
--- a/data/contracts.json.sample
+++ b/data/contracts.json.sample
@ -33,15 +33,5 @@
        "contract_address": "0x6c61fB2400Bf55624ce15104e00F269102dC2Af4",
        "erc1155": false,
        "start_block": 14940603
-    },
-    "nftisse": {
-        "contract_address": "0x343b68141129ec115c1fc523c5ae90586fe95b77",
-        "erc1155": false,
-        "start_block": 	15358089
-    },
-    "renascence": {
-        "contract_address": "0x501a31185927136E87cDfC97dDd4553D8eC1bb4A",
-        "erc1155": false,
-        "start_block": 	15973593
    }
 }
--- a/data/x2y2.json
+++ b/data/x2y2.json
--- a/package.json
+++ b/package.json
@ -5,7 +5,6 @@
    "node": "16.x"
  },
  "description": "",
-  "type": "commonjs",
  "main": "src/main.js",
  "scripts": {
    "serve": "node --max_old_space_size=128 --optimize_for_size src/server.js",
@ -18,7 +17,7 @@
  "author": "lza_menace",
  "license": "ISC",
  "dependencies": {
-    "better-sqlite3": "^11.5.0",
+    "better-sqlite3": "^7.4.5",
    "bignumber.js": "^9.0.1",
    "concurrently": "^6.5.0",
    "dotenv": "^10.0.0",
--- a/src/cli.js
+++ b/src/cli.js
@ -1,23 +0,0 @@
-const fs = require('fs');
-
-const ALL_CONTRACTS = require('../data/contracts');
-const BLOCKS_PER_HOUR = 300;
-const ARGS = process.argv.slice(2);
-
-switch (ARGS[0]) {
-    case 'rewind':
-        if (isNaN(ARGS[1])) {
-          console.log('Invalid argument passed. Provide the number of hours you would like to rewind syncing to.');
-          break;
-        }
-        const rewindBlocks = BLOCKS_PER_HOUR * Number(ARGS[1]);
-        for(const key in ALL_CONTRACTS) {
-            if (process.env.ONLY && process.env.ONLY != key) continue
-            const lastFile = `./storage/lastBlock.${key}.txt`;
-            const currentBlock = fs.readFileSync(lastFile);
-            const newBlock = Number(currentBlock) - rewindBlocks;
-            console.log(`Rewinding ${lastFile} ${rewindBlocks} blocks (${newBlock})`);
-            fs.writeFileSync(lastFile, newBlock.toString());
-        }
-        break;
-}
--- a/src/dbmgmt.js
+++ b/src/dbmgmt.js
@ -1,47 +0,0 @@
-const Database = require('better-sqlite3');
-const fs = require('fs');
-
-if (fs.existsSync('.env.local')) {
-  require('dotenv').config({path: '.env.local'});
-} else {
-  console.warn('[!] No .env.local found, quitting.');
-  process.exit();
-}
-
-const db = new Database('./storage/sqlite.db');
-const backupPath = './storage/backup.json';
-
-let l = process.argv.filter((val, idx, arr) => idx == 2)[0]
-if (l == 'backup') {
-  console.log(`performing backup of SQLite data to ${backupPath}`);
-  const results = [];
-  const stmt = db.prepare(`SELECT * FROM events ORDER BY tx_date DESC`);
-  for (const entry of stmt.iterate()) {
-    results.push(entry);
-  }
-  fs.writeFileSync(backupPath, JSON.stringify(results));
-  console.log(`[+] Wrote ${results.length} records to ${backupPath}`);
-} else if (l == 'restore') {
-  console.log(`restoring backup of SQLite data from ${backupPath}`);
-  const backupData = require('../storage/backup.json');
-  console.log(`deleting old data first`);
-  const deleteEvents = db.prepare(`DELETE FROM events where event_type != 'yolo'`);
-  deleteEvents.run();
-  console.log(`inserting new data from backup file`)
-  const insertEvent = db.prepare('INSERT INTO events (contract, event_type, from_wallet, to_wallet, token_id, amount, tx_date, tx, log_index, platform, discord_sent, twitter_sent) VALUES (@contract, @event_type, @from_wallet, @to_wallet, @token_id, @amount, @tx_date, @tx, @log_index, @platform, @discord_sent, @twitter_sent)');
-  const insertEvents = db.transaction((events) => {
-    for (let ev of events) {
-      if (!ev.discord_sent) {
-        ev.discord_sent = 0;
-      }
-      if (!ev.twitter_sent) {
-        ev.twitter_sent = 0;
-      }
-      insertEvent.run(ev)
-    };
-  });
-  insertEvents(backupData);
-} else {
-  console.log(`[!] Invalid arguments provided, quitting!`)
-  process.exit();
-}
--- a/src/poster.js
+++ b/src/poster.js
@ -13,13 +13,14 @@ const assetsBase = 'https://art101-assets.s3.us-west-2.amazonaws.com';

 async function postDiscord(_q) {
  if (process.env.DISCORD_ACTIVE == 0) return
+  const contractAddress = ethers.utils.getAddress(_q.contractAddress);
  try {
    const title = `Sale of token ${_q.tokenId} for ${_q.contractName}!`;
-    const desc = `Purchased by ${shortenAddress(_q.targetOwner)} at <t:${Number(_q.txDate.getTime()) / 1000}> for ${ethers.utils.formatEther(_q.amount.toString())}Ξ on ${camelCase(_q.eventSource)}. [Etherscan](https://etherscan.io/tx/${_q.txHash})`;
-    const url = `${assetsBase}/${_q.contractAddress}/${_q.tokenId.toString()}.json`;
+    const desc = `Purchased by ${shortenAddress(_q.targetOwner)} at <t:${Number(_q.txDate.getTime()) / 1000}> for ${ethers.utils.formatEther(_q.amount.toString())}Ξ on ${camelCase(_q.eventSource)}`;
+    const url = `${assetsBase}/${contractAddress}/${_q.tokenId.toString()}.json`;
    const metadata = await fetch(url)
      .then((r) => r.json());
-    const imageURL = metadata.image.replace('ipfs://', `${assetsBase}/${_q.contractAddress}/`) + '.fullsize.png';
+    const imageURL = metadata.image.replace('ipfs://', `${assetsBase}/${contractAddress}/`);
    const res = await fetch(process.env.DISCORD_WEBHOOK, {
      method: 'POST',
      headers: {
@ -34,12 +35,11 @@ async function postDiscord(_q) {
            image: {
              url: imageURL
            },
-            url: `https://gallery.art101.io/collection/${_q.contractName}/${_q.tokenId}`
+            url: `https://etherscan.io/tx/${_q.txHash}`
          }
        ]
      })
    });
-    return `posted sale info to Discord: ${title} - ${desc} - ${imageURL}`;
  } catch(err) {
    throw new Error(`[!] Failed to post to Discord: ${err}`);
  }
--- a/src/scraper.js
+++ b/src/scraper.js
@ -1,5 +1,5 @@
-const fs = require('fs');
 const { BigNumber, ethers } = require('ethers');
+const fs = require('fs');
 const { Database } = require('sqlite3');
 const { postDiscord } = require('./poster');

@ -15,10 +15,11 @@ const CHUNK_SIZE = Number(process.env.CHUNK_SIZE);
 const ALL_CONTRACTS = require('../data/contracts');
 const ERC721_ABI = require('../data/erc721');
 const ERC1155_ABI = require('../data/erc1155');
+const MARKETPLACE_ABI = require('../data/marketplace');
 const SEAPORT_ABI = require('../data/seaport');
 const WYVERN_ABI = require('../data/wyvern');
 const LOOKSRARE_ABI = require('../data/looksrare');
-const BLUR_ABI = require('../data/blur');
+const X2Y2_ABI = require('../data/x2y2');
 const WETH_ADDRESS = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2';
 const TRANSFER_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef';
 const TRANSFER_SINGLE_TOPIC = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62';
@ -26,11 +27,10 @@ const LOOKSRARE_SALE_TOPIC = '0x95fb6205e23ff6bda16a2d1dba56b9ad7c783f67c96fa149
 const SEAPORT_SALE_TOPIC = '0x9d9af8e38d66c62e2c12f0225249fd9d721c54b83f48d9352c97c6cacdcb6f31';
 const WYVERN_SALE_TOPIC = '0xc4109843e0b7d514e4c093114b863f8e7d8d9a458c372cd51bfe526b588006c9'
 const X2Y2_SALE_TOPIC = '0x3cbb63f144840e5b1b0a38a7c19211d2e89de4d7c5faf8b2d3c1776c302d1d33';
-const BLUR_SALE_TOPIC = '0x61cbb2a3dee0b6064c2e681aadd61677fb4ef319f0b547508d495626f5a62f64';
 const seaportInterface = new ethers.utils.Interface(SEAPORT_ABI);
 const looksrareInterface = new ethers.utils.Interface(LOOKSRARE_ABI);
 const wyvernInterface = new ethers.utils.Interface(WYVERN_ABI);
-const blurInterface = new ethers.utils.Interface(BLUR_ABI);
+const x2y2Interface = new ethers.utils.Interface(X2Y2_ABI);
 const provider = new ethers.providers.WebSocketProvider(process.env.GETH_NODE);
 const db = new Database('./storage/sqlite.db');

@ -43,7 +43,7 @@ class Collection {
    }
    const data = ALL_CONTRACTS[contractName];
    this.contractName = contractName;
-    this.contractAddress = data['contract_address'];
+    this.contractAddress = data['contract_address'].toLowerCase();
    this.erc1155 = data['erc1155'];
    this.startBlock = data['start_block'];
    if (this.erc1155) {
@ -61,11 +61,10 @@ class Scrape extends Collection {

  provider = this.getWeb3Provider();

-  constructor (contractName, blockNumber) {
+  constructor (contractName) {
    super(contractName);
    this.contract = new ethers.Contract(this.contractAddress, this.abi, this.provider);
    this.lastFile = `./storage/lastBlock.${this.contractName}.txt`;
-    this.lastBlock = blockNumber;
    createDatabaseIfNeeded();
  }

@ -76,37 +75,43 @@ class Scrape extends Collection {

  // continuous scanning loop
  async scrape() {
-    let latestEthBlock = this.lastBlock;
+    let latestEthBlock = await this.provider.getBlockNumber();
    let lastScrapedBlock = this.getLastBlock();
-    const lastRequested = lastScrapedBlock;
+    while (true) {
+      const lastRequested = lastScrapedBlock;

-    await this.filterTransfers(lastScrapedBlock).then(async ev => {
-      // capture transfer events with returned array of Transfers
-      try {
-        await this.getTransferEvents(ev);
-      } catch(err) {
-        console.log(ev)
-        throw new Error(err);
-      }
-      // filter down unique transaction hashes
-      ev.map(tx => tx.transactionHash).filter((tx, i, a) => a.indexOf(tx) === i).map(async txHash => {
-        // capture sales events for each
+      await this.filterTransfers(lastScrapedBlock).then(async ev => {
+        // capture transfer events with returned array of Transfers
        try {
-          await this.getSalesEvents(txHash);
-          await sleep(1);
+          await this.getTransferEvents(ev);
        } catch(err) {
-          console.log(txHash)
+          console.log(ev)
          throw new Error(err);
        }
+        // filter down unique transaction hashes
+        ev.map(tx => tx.transactionHash).filter((tx, i, a) => a.indexOf(tx) === i).map(async txHash => {
+          // capture sales events for each
+          try {
+            await this.getSalesEvents(txHash);
+          } catch(err) {
+            console.log(txHash)
+            throw new Error(err);
+          }
+        });
      });
-    });

-    if (lastRequested === lastScrapedBlock) {
-      lastScrapedBlock += CHUNK_SIZE;
-      this.writeLastBlock(lastScrapedBlock);
-      if (lastScrapedBlock > latestEthBlock) lastScrapedBlock = latestEthBlock;
-    }
+      if (lastRequested === lastScrapedBlock) {
+        lastScrapedBlock += CHUNK_SIZE;
+        this.writeLastBlock(lastScrapedBlock);
+        if (lastScrapedBlock > latestEthBlock) lastScrapedBlock = latestEthBlock;
+      }

+      while (lastScrapedBlock >= latestEthBlock) {
+        latestEthBlock = await this.provider.getBlockNumber();
+        console.log(`[ ${(new Date()).toISOString()} ][ ${this.contractName} ] [ waiting ]\n`)
+        await sleep(120);
+      }
+    }
  }

  // query historical logs
@ -124,8 +129,8 @@ class Scrape extends Collection {

  // get transfer events from a batch from filtering
  async getTransferEvents(txEvents) {
-    for (const tx of txEvents) {
-      await sleep(1);
+    let platform = 'contract';
+    txEvents.forEach(async tx => {
      let tokenId;
      if (this.erc1155) {
        tokenId = tx.args.id.toString();
@ -134,8 +139,7 @@ class Scrape extends Collection {
      }
      const fromAddress = tx.args.from.toString().toLowerCase();
      const toAddress = tx.args.to.toString().toLowerCase();
-      const ts = (await tx.getBlock()).timestamp
-      const timestamp = new Date(ts * 1000);
+      const timestamp = await this.getBlockTimestamp(tx.blockNumber);
      let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ transfer ] #${tokenId}: ${fromAddress} => ${toAddress} in tx ${tx.transactionHash}:${tx.logIndex}\n`;
      console.log(msg);
      const q = {
@ -152,9 +156,9 @@ class Scrape extends Collection {
        txDate: timestamp
      }
      writeToDatabase(q)
-        // .then((_) => this.writeLastBlock(tx.blockNumber))
+        .then((res) => this.writeLastBlock(tx.blockNumber))
        .catch((err) => console.log(`Error writing to database: ${err}`));
-    };
+    });
  }

  // get sales events from a given transaction
@ -162,8 +166,6 @@ class Scrape extends Collection {
    try {
      const receipt = await this.provider.getTransactionReceipt(txHash);
      const timestamp = await this.getBlockTimestamp(receipt.blockNumber);
-      const _logs = receipt.logs.filter((l) => l.address.toLowerCase() === this.contractAddress.toLowerCase());
-      if (_logs == 0) return
      // Evaluate each log entry and determine if it's a sale for our contract and use custom logic for each exchange to parse values
      receipt.logs.map(async log => {
        let logIndex = log.logIndex;
@ -171,68 +173,53 @@ class Scrape extends Collection {
        let platform;
        let fromAddress;
        let toAddress;
-        let amountWei = 0;
+        let amountWei;
+        let amountEther;
        let tokenId;
        if (log.topics[0].toLowerCase() === SEAPORT_SALE_TOPIC.toLowerCase()) {
          // Handle Opensea/Seaport sales
-          const logDescription = seaportInterface.parseLog(log);
+          const logDescription = seaportInterface.parseLog(log)
+          const matchingOffers = logDescription.args.offer.filter(
+            o =>  o.token.toLowerCase() == this.contractAddress
+          );
+          if (matchingOffers.length === 0) return;
+          sale = true;
          platform = 'opensea';
-          if (logDescription.args.offer[0].token.toLowerCase() == this.contractAddress.toLowerCase()) {
-            // buyer has accepted seller offer
-            sale = true;
-            fromAddress = logDescription.args.offerer.toLowerCase();
-            toAddress = logDescription.args.recipient.toLowerCase();
-            logDescription.args.consideration.map((o) => {
-              if (Number(o.amount) > 0) amountWei += Number(o.amount);
-            });
-            amountWei = amountWei.toString();
-            let rl = logDescription.args.offer.filter((l) => l.token.toLowerCase() === this.contractAddress.toLowerCase());
-            if (rl.length > 0) tokenId = rl[0].identifier.toString();
-          } else if (logDescription.args.offer[0].token.toLowerCase() == WETH_ADDRESS.toLowerCase()) {
-            // seller has accepted buyer bid (uses WETH)
-            // filter down only sales on the contract
-            const _c = logDescription.args.consideration.filter((c) => c.token.toLowerCase() === this.contractAddress.toLowerCase());
-            _c.map((o) => {
-              sale = true;
-              toAddress = logDescription.args.offerer.toLowerCase();
-              fromAddress = logDescription.args.recipient.toLowerCase();
-              amountWei = BigNumber.from(logDescription.args.offer[0].amount).toString();
-              tokenId = _c[0].identifier.toString();
-            })
-          } else {
-            // unknown condition
+          fromAddress = logDescription.args.offerer.toLowerCase();
+          toAddress = logDescription.args.recipient.toLowerCase();
+          tokenId = logDescription.args.offer.map(o => o.identifier.toString());
+          let amounts = logDescription.args.consideration.map(c => BigInt(c.amount));
+          // add weth
+          const wethOffers = matchingOffers.map(o => o.token.toLowerCase() === WETH_ADDRESS.toLowerCase() && o.amount > 0 ? BigInt(o.amount) : BigInt(0));
+          if (wethOffers.length > 0 && wethOffers[0] != BigInt(0)) {
+            amounts = wethOffers
          }
+          amountWei = amounts.reduce((previous,current) => previous + current, BigInt(0));
        } else if (log.topics[0].toLowerCase() === WYVERN_SALE_TOPIC.toLowerCase()) {
          // Handle Opensea/Wyvern sales
-          let txEventType = TRANSFER_TOPIC.toLowerCase();
          const logDescription = wyvernInterface.parseLog(log);
          sale = true;
          platform = 'opensea';
-          if (this.erc1155) txEventType = TRANSFER_SINGLE_TOPIC.toLowerCase();
-          // get transfer log to deduce from/to/token_id
-          const txLog = receipt.logs.filter(
-            l => (
-              l.topics[0].toLowerCase() == txEventType
-              &&
-              l.address.toLowerCase() === this.contractAddress.toLowerCase()
-              &&
-              l.logIndex === logIndex - 1 // transfer should be immediately before sale
-            )
-          );
-          if (txLog.length === 0) return;
-          const txLogDescription = this.interface.parseLog(txLog[0]);
-          fromAddress = txLogDescription.args.from.toLowerCase();
-          toAddress = txLogDescription.args.to.toLowerCase();
          if (this.erc1155) {
-            tokenId = BigNumber.from(txLogDescription.args.id).toString();
+            const txLog = receipt.logs.map(l => l).filter(_l =>
+              (_l.topics[0].toLowerCase() == TRANSFER_SINGLE_TOPIC.toLowerCase())
+            ).map(t => this.interface.parseLog(t))[0].args;
+            fromAddress = txLog.from.toLowerCase();
+            toAddress = txLog.to.toLowerCase();
+            tokenId = BigNumber.from(txLog.id).toString();
          } else {
-            tokenId = BigNumber.from(txLogDescription.args.tokenId).toString();
+            const txLog = receipt.logs.map(l => l).filter(_l =>
+              (_l.topics[0].toLowerCase() == TRANSFER_TOPIC.toLowerCase())
+            ).map(t => this.interface.parseLog(t))[0].args;
+            fromAddress = txLog.from.toLowerCase();
+            toAddress = txLog.to.toLowerCase();
+            tokenId = BigNumber.from(txLog.tokenId).toString();
          }
          amountWei = BigInt(logDescription.args.price);
        } else if (log.topics[0].toLowerCase() === LOOKSRARE_SALE_TOPIC.toLowerCase()) {
          // Handle LooksRare sales
          const logDescription = looksrareInterface.parseLog(log);
-          if (logDescription.args.collection.toLowerCase() != this.contractAddress.toLowerCase()) return;
+          if (logDescription.args.collection.toLowerCase() != this.contractAddress) return;
          sale = true;
          platform = 'looksrare';
          fromAddress = logDescription.args.maker.toLowerCase();
@ -241,6 +228,8 @@ class Scrape extends Collection {
          amountWei = logDescription.args.price.toString();
        } else if (log.topics[0].toLowerCase() === X2Y2_SALE_TOPIC.toLowerCase()) {
          // Handle x2y2 sales
+          const logDescription = x2y2Interface.parseLog(log);
+          return
          const data = log.data.substring(2);
          const dataSlices = data.match(/.{1,64}/g);
          sale = true;
@ -252,27 +241,9 @@ class Scrape extends Collection {
          if (amountWei === BigInt(0)) {
            amountWei = BigInt(`0x${dataSlices[26]}`);
          }
-        } else if (log.topics[0].toLowerCase() === BLUR_SALE_TOPIC.toLowerCase()) {
-          // Handle Blur sales
-          sale = true;
-          platform = 'blur';
-          const logDescription = blurInterface.parseLog(log);
-          fromAddress = logDescription.args.maker;
-          toAddress = logDescription.args.taker;
-          tokenId = BigInt(logDescription.args.sell.tokenId);
-          amountWei = BigInt(logDescription.args.sell.price);
-          // console.log(logDescription)
-          // Blur's marketplace orders don't include the purchaser, only their proxy contract which passes the token through
-          // This little hack just grabs the Transfer event after the Blur sale to get the end recipient
-          let rl = receipt.logs.filter(
-            l => l.logIndex === log.logIndex + 2 && l.topics[0].toLowerCase() === TRANSFER_TOPIC
-          );
-          if (rl.length > 0) {
-            toAddress = ethers.utils.defaultAbiCoder.decode(['address'], rl[0].topics[2])[0].toLowerCase();
-          }
        }
        if (sale) {
-          let amountEther = ethers.utils.formatEther(amountWei.toString());
+          amountEther = ethers.utils.formatEther(amountWei);
          let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ sale ] #${tokenId}: ${fromAddress} => ${toAddress} for ${amountEther}Ξ (${platform}) in tx ${txHash}:${logIndex}\n`;
          console.log(msg);
          const q = {
@ -284,24 +255,14 @@ class Scrape extends Collection {
            eventSource: platform,
            sourceOwner: fromAddress,
            targetOwner: toAddress,
-            tokenId: tokenId.toString(),
+            tokenId: tokenId,
            amount: amountWei,
            txDate: timestamp
          }
          writeToDatabase(q)
-            .then(async _ => {
-              let notifSent = await checkUnsentNotif(txHash, logIndex);
-              if (process.env.DISCORD_ACTIVE == 1 && (notifSent || process.env.FORCE == 1)) {
-                postDiscord(q)
-                  .then(async res => {
-                    await markSent(txHash, logIndex);
-                    console.log(`[ ${timestamp.toISOString()} ][ ${this.contractName} ][ discord ] ${res}\n`)
-                  })
-                  .catch((err) => console.log(`Error posting to Discord: ${err}`));
-              }
-              // this.writeLastBlock(log.blockNumber);
-            })
+            .then((res) => this.writeLastBlock(log.blockNumber))
            .catch((err) => console.log(`Error writing to database: ${err}`));
+          await postDiscord(q);
        }
      });
    } catch(err) {
@ -361,7 +322,7 @@ async function createDatabaseIfNeeded() {
        `CREATE TABLE events (
          contract text, event_type text, from_wallet text, to_wallet text,
          token_id number, amount number, tx_date text, tx text,
-          log_index number, platform text, discord_sent number, twitter_sent number,
+          log_index number, platform text,
          UNIQUE(tx, log_index)
        );`,
      );
@ -387,35 +348,13 @@ async function checkRowExists(txHash, logIndex) {
  return rowExists;
 }

-async function checkUnsentNotif(txHash, logIndex) {
-  const rowExists = await new Promise((resolve) => {
-    db.get('SELECT * FROM events WHERE tx = ? AND log_index = ? AND discord_sent != 1', [txHash, logIndex], (err, row) => {
-      if (err) {
-        resolve(false);
-      }
-      resolve(row !== undefined);
-    });
-  });
-  return rowExists;
-}
-
-async function markSent(txHash, logIndex) {
-  try {
-    const stmt = db.prepare('UPDATE events SET discord_sent = 1 WHERE tx = ? AND log_index = ?');
-    stmt.run(txHash, logIndex);
-    stmt.finalize();
-  } catch(err) {
-    console.log(`Error writing to database: ${err}`)
-  }
-}
-
 async function writeToDatabase(_q) {
  // txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate
  const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress);
  if (!rowExists) {
    let stmt;
    try {
-      stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)');
+      stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?)');
      stmt.run(
        _q.contractAddress,
        _q.eventName,
@ -426,9 +365,7 @@ async function writeToDatabase(_q) {
        _q.txDate.toISOString(),
        _q.txHash,
        _q.logIndex,
-        _q.eventSource,
-        0,
-        0
+        _q.eventSource
      );
      stmt.finalize();
      return true;
@ -441,24 +378,27 @@ async function writeToDatabase(_q) {
  return true;
 }

-(async () => {
-  if (process.env.TX) {
-    c.getSalesEvents(process.env.TX);
-    return
-  } else {
-    const latestBlock = await provider.getBlockNumber();
-    while(true) {
-      for(const key in ALL_CONTRACTS) {
-        if (process.env.ONLY && process.env.ONLY != key) continue
-        const c = new Scrape(key, latestBlock);
-        try {
-          await c.scrape();
-          await sleep(2);
-        } catch(e) {
-          console.log(e);
-          await sleep(10);
-        }
-      }
-    }
+// Sample events for testing functionality and detecting sales
+let c = new Scrape('mondriannft');
+// c.getSalesEvents('0x2f8961209daca23288c499449aa936b54eec5c25720b9d7499a8ee5bde7fcdc7')
+// c.getSalesEvents('0xb20853f22b367ee139fd800206bf1cba0c36f1a1dd739630f99cc6ffd0471edc')
+// c.getSalesEvents('0x71e5135a543e17cc91992a2229ae5811461c96b84d5e2560ac8db1dd99bb17e3')
+c.getSalesEvents('0xe567d00bb0c16928d5d8c258de8dd928e93209b40f7c958bc485d2a6c549b8a9')
+// c.getSalesEvents('0x975d10cdd873ee5bb29e746c2f1f3b776078cace9c04ce419cb66949239288b5')
+// c.getSalesEvents('0x8d45ed8168a740f8b182ec0dbad1c37d6c6dbd8aa865be408d865ca01fb0fa94')
+// c.getSalesEvents('0x27ab6f12604bf17a9e7c93bf1a7cc466d7dfd922565d267eac10879b59d5d0b5')
+// c.getSalesEvents('0x511bc5cda2b7145511c7b57e29cecf1f15a5a650670f09e91e69fc24824effd9')
+// c.getSalesEvents('0x04746b6ba1269906db8e0932263b86a6fc35a30a31cf73d2b7db078f6f4ed442')
+// c.getSalesEvents('0x24d6523c5048b2df3e7f8b24d63a6644e4c0ed33cfae6396190e3ded5fc79321')
+// c.getSalesEvents('0xe56dc64c44a3cbfe3a1e68f8669a65f17ebe48d64e944673122a565b7c641d1e')
+return
+
+if (process.env.SCRAPE) {
+  let c = new Scrape(process.env.SCRAPE)
+  c.scrape()
+} else {
+  for(const key in ALL_CONTRACTS) {
+    const c = new Scrape(key);
+    c.scrape();
  }
-})();
+}
--- a/src/server.js
+++ b/src/server.js
@ -10,7 +10,7 @@ if (fs.existsSync('.env.local')) {
 }

 const ALL_CONTRACTS = require('../data/contracts');
-const db = new Database('./storage/sqlite.db', {readonly: true});
+const db = new Database('./storage/sqlite.db');
 const app = express();
 const port = process.env.PORT || 3000;

@ -24,15 +24,66 @@ app.get('/api/contracts', (req, res) => {
  res.status(200).json(ALL_CONTRACTS)
 })

+app.get('/api/:contractAddress/offers', (req, res) => {
+  const results = [];
+  const stmt = db.prepare(`select *
+    from events ev
+    where contract = '${req.params.contractAddress}'
+    collate nocase
+    and event_type = 'tokenoffered'
+    and not token_id in
+    (
+      select token_id from events
+      where event_type == 'tokennolongerforsale'
+      and token_id = token_id
+      and contract = '${req.params.contractAddress}'
+      collate nocase
+      and tx_date > ev.tx_date
+      order by tx_date asc
+      limit 1
+    )
+    order by tx_date desc
+    `);
+    for (const entry of stmt.iterate()) {
+      results.push(entry);
+    }
+    res.status(200).json(results);
+});
+
+app.get('/api/:contractAddress/bids', (req, res) => {
+  const results = [];
+  const stmt = db.prepare(`select *
+    from events ev
+    where contract = '${req.params.contractAddress}'
+    collate nocase
+    and event_type = 'tokenbidentered'
+    and not token_id in
+    (
+      select token_id from events
+      where event_type == 'tokenbidwithdrawn'
+      and token_id = token_id
+      and contract = '${req.params.contractAddress}'
+      collate nocase
+      and tx_date > ev.tx_date
+      order by tx_date asc
+      limit 1
+    )
+    order by tx_date desc
+    `);
+    for (const entry of stmt.iterate()) {
+      results.push(entry);
+    }
+    res.status(200).json(results);
+});
+
 app.get('/api/:contractAddress/events', (req, res) => {
  const results = [];
  const stmt = db.prepare(`select *
    from events
    where contract = '${req.params.contractAddress}'
    collate nocase
-    and event_type == 'sale' or event_type == 'transfer'
+    and event_type != 'sale' and event_type != 'transfer'
    order by tx_date desc
-    limit 100
    `);
  for (const entry of stmt.iterate()) {
    results.push(entry);