commit 6a3584224debfed964135e8d1facfeedbd2e72a1 Author: lza_menace Date: Mon Dec 23 01:28:20 2024 -0800 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e41ab8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,177 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store + +state \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ce6e853 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# alchemy-nft-scraper + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run src/index.js +``` + +This project was created using `bun init` in bun v1.1.27. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..337d096 Binary files /dev/null and b/bun.lockb differ diff --git a/package.json b/package.json new file mode 100644 index 0000000..2aaa4fe --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "dependencies": { + "alchemy-sdk": "^3.5.0", + "ethers": "^6.13.4", + "sqlite3": "^5.1.7" + }, + "name": "alchemy-nft-scraper", + "module": "index.ts", + "type": "module", + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.7.2" + } +} \ No newline at end of file diff --git a/src/contracts.json b/src/contracts.json new file mode 100644 index 0000000..805ec17 --- /dev/null +++ b/src/contracts.json @@ -0,0 +1,42 @@ +{ + "non-fungible-soup": { + "contract_address": "0xdc8bEd466ee117Ebff8Ee84896d6aCd42170d4bB", + "erc1155": false, + "start_block": 13105421 + }, + "mondriannft": { + "contract_address": "0x7f81858ea3b43513adfaf0a20dc7b4c6ebe72919", + "erc1155": false, + "start_block": 13239362 + }, + "bauhausblocks": { + "contract_address": "0x62C1e9f6830098DFF647Ef78E1F39244258F7bF5", + "erc1155": false, + "start_block": 13439681 + }, + "nftzine": { + "contract_address": "0xc918F953E1ef2F1eD6ac6A0d2Bf711A93D20Aa2b", + "erc1155": false, + "start_block": 13698461 + }, + "basedvitalik": { + "contract_address": "0xea2dc6f116a4c3d6a15f06b4e8ad582a07c3dd9c", + "erc1155": false, + "start_block": 14254106 + }, + "rmutt": { + "contract_address": "0x6c61fB2400Bf55624ce15104e00F269102dC2Af4", + "erc1155": false, + "start_block": 14940603 + }, + "nftisse": { + "contract_address": "0x343b68141129ec115c1fc523c5ae90586fe95b77", + "erc1155": false, + "start_block": 15358089 + }, + "renascence": { + "contract_address": "0x501a31185927136E87cDfC97dDd4553D8eC1bb4A", + "erc1155": false, + "start_block": 15973593 + } +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..3b9983a --- /dev/null +++ b/src/index.js @@ -0,0 +1,145 @@ +const ALL_CONTRACTS = require('./contracts'); + +const { Alchemy, Network } = require("alchemy-sdk"); +const { Database } = require('sqlite3'); +const { ethers } = require("ethers"); +const fs = require('fs'); + + +const db = new Database('./state/sqlite.db'); +const config = { + apiKey: process.env.ALCHEMY_KEY, + network: Network.ETH_MAINNET, +}; + +const alchemy = new Alchemy(config); + + +async function sleep(sec) { + return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000)); +} + + +class Scrape { + + constructor (contractName) { + if (!(contractName in ALL_CONTRACTS)) { + console.warn(`[!] That contract name does not exist in data/contracts.json`); + process.exit(); + } + const data = ALL_CONTRACTS[contractName]; + this.contractName = contractName; + this.contractAddress = data['contract_address']; + this.erc1155 = data['erc1155']; + this.startBlock = data['start_block']; + this.lastFile = `./state/${this.contractName}.txt`; + } + + getpageKey() { + if (fs.existsSync(this.lastFile)) { + return fs.readFileSync(this.lastFile).toString(); + } else { + fs.writeFileSync(this.lastFile, ''); + return null + }; + } + + async scrape() { + const pageKey = this.getpageKey() || null + console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`) + const response = await alchemy.nft.getNftSales({ + fromBlock: this.startBlock, + contractAddress: this.contractAddress, + limit: process.env.LIMIT, + order: 'asc', + pageKey: pageKey + }); + fs.writeFileSync(this.lastFile, response.pageKey) + response.nftSales.map(async (sale) => { + const rowExists = await new Promise((resolve) => { + db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => { + if (err) { resolve(false); } + resolve(row !== undefined); + }); + }); + if (!rowExists) { + try { + db.run(` + INSERT INTO events VALUES ( + "${sale.contractAddress}", + "${sale.buyerAddress}", + "${sale.sellerAddress}", + "${sale.taker}", + "${sale.tokenId}", + "${sale.sellerFee.amount}", + "${sale.protocolFee.amount}", + "${sale.royaltyFee.amount}", + "", + "${sale.transactionHash}", + "${sale.blockNumber}", + "${sale.logIndex}", + "${sale.bundleIndex}", + "${sale.marketplace}", + 0, 0 + )`); + console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`) + } catch(err) { + console.log(`Error when writing to database: ${err}`); + return false; + } + } + }); + + await sleep(1); + + } + +} + +(async () => { + const tableExists = await new Promise((resolve) => { + db.get('SELECT name FROM sqlite_master WHERE type="table" AND name="events"', [], (err, row) => { + if (err) { + resolve(false); + } + resolve(row !== undefined); + }); + }); + if (!tableExists) { + db.serialize(() => { + db.run( + `CREATE TABLE events ( + contract text, + buyer text, + seller text, + taker text, + token_id number, + sale_price text, + protocol_fee text, + royalty_fee text, + tx_date text, + tx_hash text, + block_number number, + log_index number, + bundle_index number, + marketplace text, + discord_sent number, + twitter_sent number, + UNIQUE(tx_hash, log_index, bundle_index) + );`, + ); + }); + } + while(true) { + for(const contract in ALL_CONTRACTS) { + if (process.env.ONLY && process.env.ONLY != contract) continue + const c = new Scrape(contract); + try { + await c.scrape(); + } catch(e) { + console.log(e); + } + await sleep(3); + } + } +})(); \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..238655f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}