init
commit
6a3584224d
@ -0,0 +1,177 @@
|
||||
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
|
||||
|
||||
# Logs
|
||||
|
||||
logs
|
||||
_.log
|
||||
npm-debug.log_
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Caches
|
||||
|
||||
.cache
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# Runtime data
|
||||
|
||||
pids
|
||||
_.pid
|
||||
_.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
|
||||
.temp
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.*
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
|
||||
state
|
||||
@ -0,0 +1,15 @@
|
||||
# alchemy-nft-scraper
|
||||
|
||||
To install dependencies:
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
To run:
|
||||
|
||||
```bash
|
||||
bun run src/index.js
|
||||
```
|
||||
|
||||
This project was created using `bun init` in bun v1.1.27. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.
|
||||
@ -0,0 +1,16 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"alchemy-sdk": "^3.5.0",
|
||||
"ethers": "^6.13.4",
|
||||
"sqlite3": "^5.1.7"
|
||||
},
|
||||
"name": "alchemy-nft-scraper",
|
||||
"module": "index.ts",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5.7.2"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
{
|
||||
"non-fungible-soup": {
|
||||
"contract_address": "0xdc8bEd466ee117Ebff8Ee84896d6aCd42170d4bB",
|
||||
"erc1155": false,
|
||||
"start_block": 13105421
|
||||
},
|
||||
"mondriannft": {
|
||||
"contract_address": "0x7f81858ea3b43513adfaf0a20dc7b4c6ebe72919",
|
||||
"erc1155": false,
|
||||
"start_block": 13239362
|
||||
},
|
||||
"bauhausblocks": {
|
||||
"contract_address": "0x62C1e9f6830098DFF647Ef78E1F39244258F7bF5",
|
||||
"erc1155": false,
|
||||
"start_block": 13439681
|
||||
},
|
||||
"nftzine": {
|
||||
"contract_address": "0xc918F953E1ef2F1eD6ac6A0d2Bf711A93D20Aa2b",
|
||||
"erc1155": false,
|
||||
"start_block": 13698461
|
||||
},
|
||||
"basedvitalik": {
|
||||
"contract_address": "0xea2dc6f116a4c3d6a15f06b4e8ad582a07c3dd9c",
|
||||
"erc1155": false,
|
||||
"start_block": 14254106
|
||||
},
|
||||
"rmutt": {
|
||||
"contract_address": "0x6c61fB2400Bf55624ce15104e00F269102dC2Af4",
|
||||
"erc1155": false,
|
||||
"start_block": 14940603
|
||||
},
|
||||
"nftisse": {
|
||||
"contract_address": "0x343b68141129ec115c1fc523c5ae90586fe95b77",
|
||||
"erc1155": false,
|
||||
"start_block": 15358089
|
||||
},
|
||||
"renascence": {
|
||||
"contract_address": "0x501a31185927136E87cDfC97dDd4553D8eC1bb4A",
|
||||
"erc1155": false,
|
||||
"start_block": 15973593
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,145 @@
|
||||
const ALL_CONTRACTS = require('./contracts');
|
||||
|
||||
const { Alchemy, Network } = require("alchemy-sdk");
|
||||
const { Database } = require('sqlite3');
|
||||
const { ethers } = require("ethers");
|
||||
const fs = require('fs');
|
||||
|
||||
|
||||
const db = new Database('./state/sqlite.db');
|
||||
const config = {
|
||||
apiKey: process.env.ALCHEMY_KEY,
|
||||
network: Network.ETH_MAINNET,
|
||||
};
|
||||
|
||||
const alchemy = new Alchemy(config);
|
||||
|
||||
|
||||
async function sleep(sec) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000));
|
||||
}
|
||||
|
||||
|
||||
class Scrape {
|
||||
|
||||
constructor (contractName) {
|
||||
if (!(contractName in ALL_CONTRACTS)) {
|
||||
console.warn(`[!] That contract name does not exist in data/contracts.json`);
|
||||
process.exit();
|
||||
}
|
||||
const data = ALL_CONTRACTS[contractName];
|
||||
this.contractName = contractName;
|
||||
this.contractAddress = data['contract_address'];
|
||||
this.erc1155 = data['erc1155'];
|
||||
this.startBlock = data['start_block'];
|
||||
this.lastFile = `./state/${this.contractName}.txt`;
|
||||
}
|
||||
|
||||
getpageKey() {
|
||||
if (fs.existsSync(this.lastFile)) {
|
||||
return fs.readFileSync(this.lastFile).toString();
|
||||
} else {
|
||||
fs.writeFileSync(this.lastFile, '');
|
||||
return null
|
||||
};
|
||||
}
|
||||
|
||||
async scrape() {
|
||||
const pageKey = this.getpageKey() || null
|
||||
console.log(`[+] Scraping ${this.contractName} with pageKey ${pageKey}`)
|
||||
const response = await alchemy.nft.getNftSales({
|
||||
fromBlock: this.startBlock,
|
||||
contractAddress: this.contractAddress,
|
||||
limit: process.env.LIMIT,
|
||||
order: 'asc',
|
||||
pageKey: pageKey
|
||||
});
|
||||
fs.writeFileSync(this.lastFile, response.pageKey)
|
||||
response.nftSales.map(async (sale) => {
|
||||
const rowExists = await new Promise((resolve) => {
|
||||
db.get('SELECT * FROM events WHERE tx_hash = ? AND log_index = ?', [sale.transactionHash, sale.logIndex], (err, row) => {
|
||||
if (err) { resolve(false); }
|
||||
resolve(row !== undefined);
|
||||
});
|
||||
});
|
||||
if (!rowExists) {
|
||||
try {
|
||||
db.run(`
|
||||
INSERT INTO events VALUES (
|
||||
"${sale.contractAddress}",
|
||||
"${sale.buyerAddress}",
|
||||
"${sale.sellerAddress}",
|
||||
"${sale.taker}",
|
||||
"${sale.tokenId}",
|
||||
"${sale.sellerFee.amount}",
|
||||
"${sale.protocolFee.amount}",
|
||||
"${sale.royaltyFee.amount}",
|
||||
"",
|
||||
"${sale.transactionHash}",
|
||||
"${sale.blockNumber}",
|
||||
"${sale.logIndex}",
|
||||
"${sale.bundleIndex}",
|
||||
"${sale.marketplace}",
|
||||
0, 0
|
||||
)`);
|
||||
console.log(` ::: Inserted sale of ${this.contractName} #${sale.tokenId} in block ${sale.blockNumber} for ${sale.sellerFee.amount} wei.`)
|
||||
} catch(err) {
|
||||
console.log(`Error when writing to database: ${err}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await sleep(1);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const tableExists = await new Promise((resolve) => {
|
||||
db.get('SELECT name FROM sqlite_master WHERE type="table" AND name="events"', [], (err, row) => {
|
||||
if (err) {
|
||||
resolve(false);
|
||||
}
|
||||
resolve(row !== undefined);
|
||||
});
|
||||
});
|
||||
if (!tableExists) {
|
||||
db.serialize(() => {
|
||||
db.run(
|
||||
`CREATE TABLE events (
|
||||
contract text,
|
||||
buyer text,
|
||||
seller text,
|
||||
taker text,
|
||||
token_id number,
|
||||
sale_price text,
|
||||
protocol_fee text,
|
||||
royalty_fee text,
|
||||
tx_date text,
|
||||
tx_hash text,
|
||||
block_number number,
|
||||
log_index number,
|
||||
bundle_index number,
|
||||
marketplace text,
|
||||
discord_sent number,
|
||||
twitter_sent number,
|
||||
UNIQUE(tx_hash, log_index, bundle_index)
|
||||
);`,
|
||||
);
|
||||
});
|
||||
}
|
||||
while(true) {
|
||||
for(const contract in ALL_CONTRACTS) {
|
||||
if (process.env.ONLY && process.env.ONLY != contract) continue
|
||||
const c = new Scrape(contract);
|
||||
try {
|
||||
await c.scrape();
|
||||
} catch(e) {
|
||||
console.log(e);
|
||||
}
|
||||
await sleep(3);
|
||||
}
|
||||
}
|
||||
})();
|
||||
@ -0,0 +1,27 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
// Enable latest features
|
||||
"lib": ["ESNext", "DOM"],
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
// Best practices
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
|
||||
// Some stricter flags (disabled by default)
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue