init

.gitignore

@@ -0,0 +1,177 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store
+
+state

README.md

@@ -0,0 +1,15 @@
+# alchemy-nft-scraper
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run src/index.js
+```
+
+This project was created using `bun init` in bun v1.1.27. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

package.json

@@ -0,0 +1,16 @@
+{
+  "dependencies": {
+    "alchemy-sdk": "^3.5.0",
+    "ethers": "^6.13.4",
+    "sqlite3": "^5.1.7"
+  },
+  "name": "alchemy-nft-scraper",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.7.2"
+  }
+}

src/contracts.json

@@ -0,0 +1,42 @@
+{
+    "non-fungible-soup": {
+        "contract_address": "0xdc8bEd466ee117Ebff8Ee84896d6aCd42170d4bB",
+        "erc1155": false,
+        "start_block": 13105421
+    },
+    "mondriannft": {
+        "contract_address": "0x7f81858ea3b43513adfaf0a20dc7b4c6ebe72919",
+        "erc1155": false,
+        "start_block": 13239362
+    },
+    "bauhausblocks": {
+        "contract_address": "0x62C1e9f6830098DFF647Ef78E1F39244258F7bF5",
+        "erc1155": false,
+        "start_block": 13439681
+    },
+    "nftzine": {
+        "contract_address": "0xc918F953E1ef2F1eD6ac6A0d2Bf711A93D20Aa2b",
+        "erc1155": false,
+        "start_block": 13698461
+    },
+    "basedvitalik": {
+        "contract_address": "0xea2dc6f116a4c3d6a15f06b4e8ad582a07c3dd9c",
+        "erc1155": false,
+        "start_block": 14254106
+    },
+    "rmutt": {
+        "contract_address": "0x6c61fB2400Bf55624ce15104e00F269102dC2Af4",
+        "erc1155": false,
+        "start_block": 14940603
+    },
+    "nftisse": {
+        "contract_address": "0x343b68141129ec115c1fc523c5ae90586fe95b77",
+        "erc1155": false,
+        "start_block": 	15358089
+    },
+    "renascence": {
+        "contract_address": "0x501a31185927136E87cDfC97dDd4553D8eC1bb4A",
+        "erc1155": false,
+        "start_block": 	15973593
+    }
+}

src/index.js

@@ -0,0 +1,145 @@
+const ALL_CONTRACTS = require('./contracts');
+
+const { Alchemy, Network } = require("alchemy-sdk");
+const { Database } = require('sqlite3');
+const { ethers } = require("ethers");
+const fs = require('fs');
+
+
+const db = new Database('./state/sqlite.db');
+const config = {
+    apiKey: process.env.ALCHEMY_KEY,
+    network: Network.ETH_MAINNET,
+};
+
+const alchemy = new Alchemy(config);
+
+
+async function sleep(sec) {
+  return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000));
+}
+
+
+class Scrape {
+
+  constructor (contractName) {
+    if (!(contractName in ALL_CONTRACTS)) {
+      console.warn(`[!] That contract name does not exist in data/contracts.json`);
+      process.exit();
+    }
+    const data = ALL_CONTRACTS[contractName];
+    this.contractName = contractName;
+    this.contractAddress = data['contract_address'];
+    this.erc1155 = data['erc1155'];
+    this.startBlock = data['start_block'];
+    this.lastFile = `./state/${this.contractName}.txt`;
+  }
+
+  getpageKey() {
+    if (fs.existsSync(this.lastFile)) {
+      return fs.readFileSync(this.lastFile).toString();
+    } else {
+      fs.writeFileSync(this.lastFile, '');
+      return null
+    };
+  }
+
+  async scrape() {
+    const pageKey = this.getpageKey() || null
+    console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`)
+    const response = await alchemy.nft.getNftSales({
+        fromBlock: this.startBlock,
+        contractAddress: this.contractAddress,
+        limit: process.env.LIMIT,
+        order: 'asc',
+        pageKey: pageKey
+    });
+    fs.writeFileSync(this.lastFile, response.pageKey)
+    response.nftSales.map(async (sale) => {
+      const rowExists = await new Promise((resolve) => {
+        db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => {
+          if (err) { resolve(false); }
+          resolve(row !== undefined);
+        });
+      });
+      if (!rowExists) {
+        try {
+          db.run(`
+            INSERT INTO events VALUES (
+            "${sale.contractAddress}",
+            "${sale.buyerAddress}",
+            "${sale.sellerAddress}",
+            "${sale.taker}",
+            "${sale.tokenId}",
+            "${sale.sellerFee.amount}",
+            "${sale.protocolFee.amount}",
+            "${sale.royaltyFee.amount}",
+            "",
+            "${sale.transactionHash}",
+            "${sale.blockNumber}",
+            "${sale.logIndex}",
+            "${sale.bundleIndex}",
+            "${sale.marketplace}",
+            0, 0
+          )`);
+          console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`)
+        } catch(err) {
+          console.log(`Error when writing to database: ${err}`);
+          return false;
+        }
+      }
+    });
+
+    await sleep(1);
+
+  }
+
+}
+
+(async () => {
+  const tableExists = await new Promise((resolve) => {
+    db.get('SELECT name FROM sqlite_master WHERE type="table" AND name="events"', [], (err, row) => {
+      if (err) {
+        resolve(false);
+      }
+      resolve(row !== undefined);
+    });
+  });
+  if (!tableExists) {
+    db.serialize(() => {
+      db.run(
+        `CREATE TABLE events (
+          contract text,
+          buyer text,
+          seller text,
+          taker text,
+          token_id number,
+          sale_price text,
+          protocol_fee text,
+          royalty_fee text,
+          tx_date text,
+          tx_hash text,
+          block_number number,
+          log_index number,
+          bundle_index number,
+          marketplace text,
+          discord_sent number,
+          twitter_sent number,
+          UNIQUE(tx_hash, log_index, bundle_index)
+        );`,
+      );
+    });
+  }
+  while(true) {
+    for(const contract in ALL_CONTRACTS) {
+      if (process.env.ONLY && process.env.ONLY != contract) continue
+      const c = new Scrape(contract);
+      try {
+        await c.scrape();
+      } catch(e) {
+        console.log(e);
+      }
+      await sleep(3);
+    }
+  }
+})();

tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}