x2y2-interface
lza_menace 2 years ago
commit 6b24c6ae65

@ -0,0 +1,3 @@
node_modules
db.db
storage

@ -0,0 +1,4 @@
GETH_NODE="alchemy, infura or local geth node endpoint"
CHUNK_SIZE=600
DISCORD_WEBHOOK=https://discord.com/api/webhooks/yyyy/xxxxx
DISCORD_ACTIVE=0

5
.gitignore vendored

@ -0,0 +1,5 @@
.env.local
node_modules
storage/*.txt
storage/*.db
data/contracts.json

@ -0,0 +1,8 @@
FROM node:16-stretch-slim
RUN mkdir /app
COPY package.json /app
COPY package-lock.json /app
RUN cd /app && npm i
COPY . /app
WORKDIR /app
ENTRYPOINT ["npm","run","start"]

@ -0,0 +1,53 @@
# NFT Sales Scraper
This repo contains JavaScript code to scrape the Ethereum chain for sales for any number of ERC-721 or ERC-1155 compliant tokens. It also provides a simple website and API for visualizing sales and querying information across contracts and tokens, as well as posts notifications to Discord (and eventually Twitter).
## How It Works
`scraper.js` parses a list of contracts in [data/contracts.json](data/contracts.json.sample) with some user provided metadata (if ERC-1155, contract deployed block, contract address, etc) and begins an asynchronous loop to start scraping the chain at the last checked block. The script contains topic strings for relevant transfer events for ERC-721 and ERC-1155, as well as sales events from OpenSea (Wyvern and Seaport), LooksRare, and X2Y2 exchange contracts.
When the script finds a transfer event it stores the relevant transfer data (from, to, txHash, log index, contract, etc) in a SQLite database, in addition, it checks the transaction in which a transfer event occurred and parses the event logs for any sales that may have taken place.
A record of the block numbers checked is stored locally in `./storage` so that scanning resumes
## Data
The extracted data is structured the following way in the SQLite database:
```
------------------
events
------------------
contract TEXT
event_type TEXT
from_wallet TEXT
to_wallet TEXT
token_id NUMBER
amount NUMBER
tx_date TEXT
tx TEXT
platform TEXT
```
## Setup
### Secrets
Copy the `.env` file to `.env.local` to setup your local configuration, you'll need a geth node (Infura and Alchemy provide this with good free tiers). You can optionally provide a Discord webhook URL and turn on Discord posting.
### Contracts
Copy `data/contracts.json.sample` to `data/contracts.json` and modify it for the contracts you want to scrape. Be sure to define if the contract is ERC-721 or ERC-1155 to use the proper ABI and event source. Check Etherscan or some transaction explorer to get the block number in which the contract was deployed so your scraping can start at the beginning of that contract's existence.
## API
An API that serves the scraped data is implemented in the `src/server.js` file, for now, it serves a few endpoints:
* `/api/contracts` - parses the `data/contracts.json` file to return stored contract details.
* `/api/token/:contractAddress/:tokenId/history` - queries the SQLite database to return events for ${tokenId} in ${contractAddress} passed in the URL.
* `/api/latest` - queries the SQLite database to return the latest event (limited to 1).
* `/api/:contractAddress/data` - queries the SQLite database to return sales events from ${contractAddress} passed in the URL.
* `/api/:contractAddress/platforms` - queries the SQLite database to return sales events based upon the platform where the sale took place from ${contractAddress} passed in the URL.
You can start it using `npm run serve` or `npm start`, the latter of which will concurrently start the scraping processes as well as the web server.
The root of the web service is a simple representation of the sales events using [chart.js](https://chartjs.org/) and the above API.

@ -0,0 +1,37 @@
{
"non-fungible-soup": {
"contract_address": "0xdc8bEd466ee117Ebff8Ee84896d6aCd42170d4bB",
"erc1155": false,
"start_block": 13105421
},
"mondriannft": {
"contract_address": "0x7f81858ea3b43513adfaf0a20dc7b4c6ebe72919",
"erc1155": false,
"start_block": 13239362
},
"soupxmondrian": {
"contract_address": "0x0dD0CFeAE058610C88a87Da2D9fDa496cFadE108",
"erc1155": true,
"start_block": 13343992
},
"bauhausblocks": {
"contract_address": "0x62C1e9f6830098DFF647Ef78E1F39244258F7bF5",
"erc1155": false,
"start_block": 13439681
},
"nftzine": {
"contract_address": "0xc918F953E1ef2F1eD6ac6A0d2Bf711A93D20Aa2b",
"erc1155": false,
"start_block": 13698461
},
"basedvitalik": {
"contract_address": "0xea2dc6f116a4c3d6a15f06b4e8ad582a07c3dd9c",
"erc1155": false,
"start_block": 14254106
},
"rmutt": {
"contract_address": "0x6c61fB2400Bf55624ce15104e00F269102dC2Af4",
"erc1155": false,
"start_block": 14940603
}
}

@ -0,0 +1,371 @@
[
{
"inputs": [],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "account",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"indexed": false,
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "ApprovalForAll",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "previousOwner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "OwnershipTransferred",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "from",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "to",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256[]",
"name": "ids",
"type": "uint256[]"
},
{
"indexed": false,
"internalType": "uint256[]",
"name": "values",
"type": "uint256[]"
}
],
"name": "TransferBatch",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "from",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "to",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "id",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "value",
"type": "uint256"
}
],
"name": "TransferSingle",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": false,
"internalType": "string",
"name": "value",
"type": "string"
},
{
"indexed": true,
"internalType": "uint256",
"name": "id",
"type": "uint256"
}
],
"name": "URI",
"type": "event"
},
{
"inputs": [
{
"internalType": "address",
"name": "account",
"type": "address"
},
{
"internalType": "uint256",
"name": "id",
"type": "uint256"
}
],
"name": "balanceOf",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address[]",
"name": "accounts",
"type": "address[]"
},
{
"internalType": "uint256[]",
"name": "ids",
"type": "uint256[]"
}
],
"name": "balanceOfBatch",
"outputs": [
{
"internalType": "uint256[]",
"name": "",
"type": "uint256[]"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "account",
"type": "address"
},
{
"internalType": "address",
"name": "operator",
"type": "address"
}
],
"name": "isApprovedForAll",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "owner",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "renounceOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256[]",
"name": "ids",
"type": "uint256[]"
},
{
"internalType": "uint256[]",
"name": "amounts",
"type": "uint256[]"
},
{
"internalType": "bytes",
"name": "data",
"type": "bytes"
}
],
"name": "safeBatchTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "id",
"type": "uint256"
},
{
"internalType": "uint256",
"name": "amount",
"type": "uint256"
},
{
"internalType": "bytes",
"name": "data",
"type": "bytes"
}
],
"name": "safeTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "setApprovalForAll",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "bytes4",
"name": "interfaceId",
"type": "bytes4"
}
],
"name": "supportsInterface",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "transferOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"name": "uri",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function"
}
]

@ -0,0 +1,491 @@
[
{
"inputs": [],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "approved",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "Approval",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"indexed": false,
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "ApprovalForAll",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "previousOwner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "OwnershipTransferred",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "from",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "to",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "Transfer",
"type": "event"
},
{
"inputs": [
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "approve",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "owner",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "baseURI",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "getApproved",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"internalType": "address",
"name": "operator",
"type": "address"
}
],
"name": "isApprovedForAll",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "name",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "owner",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "ownerOf",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "renounceOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "safeTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
},
{
"internalType": "bytes",
"name": "_data",
"type": "bytes"
}
],
"name": "safeTransferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "operator",
"type": "address"
},
{
"internalType": "bool",
"name": "approved",
"type": "bool"
}
],
"name": "setApprovalForAll",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [],
"name": "symbol",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "uint256",
"name": "index",
"type": "uint256"
}
],
"name": "tokenByIndex",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "address",
"name": "owner",
"type": "address"
},
{
"internalType": "uint256",
"name": "index",
"type": "uint256"
}
],
"name": "tokenOfOwnerByIndex",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [],
"name": "totalSupply",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "address",
"name": "from",
"type": "address"
},
{
"internalType": "address",
"name": "to",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "transferFrom",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "transferOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
}
],
"name": "tokenURI",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "bytes4",
"name": "interfaceId",
"type": "bytes4"
}
],
"name": "supportsInterface",
"outputs": [
{
"internalType": "bool",
"name": "",
"type": "bool"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
},
{
"inputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"name": "uri",
"outputs": [
{
"internalType": "string",
"name": "",
"type": "string"
}
],
"stateMutability": "view",
"type": "function",
"constant": true
}
]

@ -0,0 +1,690 @@
[
{
"inputs": [
{
"internalType": "address",
"name": "_currencyManager",
"type": "address"
},
{
"internalType": "address",
"name": "_executionManager",
"type": "address"
},
{
"internalType": "address",
"name": "_royaltyFeeManager",
"type": "address"
},
{ "internalType": "address", "name": "_WETH", "type": "address" },
{
"internalType": "address",
"name": "_protocolFeeRecipient",
"type": "address"
}
],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "user",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "newMinNonce",
"type": "uint256"
}
],
"name": "CancelAllOrders",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "user",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256[]",
"name": "orderNonces",
"type": "uint256[]"
}
],
"name": "CancelMultipleOrders",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "currencyManager",
"type": "address"
}
],
"name": "NewCurrencyManager",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "executionManager",
"type": "address"
}
],
"name": "NewExecutionManager",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "protocolFeeRecipient",
"type": "address"
}
],
"name": "NewProtocolFeeRecipient",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "royaltyFeeManager",
"type": "address"
}
],
"name": "NewRoyaltyFeeManager",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "transferSelectorNFT",
"type": "address"
}
],
"name": "NewTransferSelectorNFT",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "previousOwner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "OwnershipTransferred",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collection",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "royaltyRecipient",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "currency",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "amount",
"type": "uint256"
}
],
"name": "RoyaltyPayment",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": false,
"internalType": "bytes32",
"name": "orderHash",
"type": "bytes32"
},
{
"indexed": false,
"internalType": "uint256",
"name": "orderNonce",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "taker",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "maker",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "strategy",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "currency",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "collection",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "amount",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "price",
"type": "uint256"
}
],
"name": "TakerAsk",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": false,
"internalType": "bytes32",
"name": "orderHash",
"type": "bytes32"
},
{
"indexed": false,
"internalType": "uint256",
"name": "orderNonce",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "taker",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "maker",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "strategy",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "currency",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "collection",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "tokenId",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "amount",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "price",
"type": "uint256"
}
],
"name": "TakerBid",
"type": "event"
},
{
"inputs": [],
"name": "DOMAIN_SEPARATOR",
"outputs": [{ "internalType": "bytes32", "name": "", "type": "bytes32" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "WETH",
"outputs": [{ "internalType": "address", "name": "", "type": "address" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{ "internalType": "uint256", "name": "minNonce", "type": "uint256" }
],
"name": "cancelAllOrdersForSender",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256[]",
"name": "orderNonces",
"type": "uint256[]"
}
],
"name": "cancelMultipleMakerOrders",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [],
"name": "currencyManager",
"outputs": [
{
"internalType": "contract ICurrencyManager",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "executionManager",
"outputs": [
{
"internalType": "contract IExecutionManager",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{ "internalType": "address", "name": "user", "type": "address" },
{ "internalType": "uint256", "name": "orderNonce", "type": "uint256" }
],
"name": "isUserOrderNonceExecutedOrCancelled",
"outputs": [{ "internalType": "bool", "name": "", "type": "bool" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "taker", "type": "address" },
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" }
],
"internalType": "struct OrderTypes.TakerOrder",
"name": "takerBid",
"type": "tuple"
},
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "signer", "type": "address" },
{
"internalType": "address",
"name": "collection",
"type": "address"
},
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{ "internalType": "uint256", "name": "amount", "type": "uint256" },
{ "internalType": "address", "name": "strategy", "type": "address" },
{ "internalType": "address", "name": "currency", "type": "address" },
{ "internalType": "uint256", "name": "nonce", "type": "uint256" },
{ "internalType": "uint256", "name": "startTime", "type": "uint256" },
{ "internalType": "uint256", "name": "endTime", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" },
{ "internalType": "uint8", "name": "v", "type": "uint8" },
{ "internalType": "bytes32", "name": "r", "type": "bytes32" },
{ "internalType": "bytes32", "name": "s", "type": "bytes32" }
],
"internalType": "struct OrderTypes.MakerOrder",
"name": "makerAsk",
"type": "tuple"
}
],
"name": "matchAskWithTakerBid",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "taker", "type": "address" },
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" }
],
"internalType": "struct OrderTypes.TakerOrder",
"name": "takerBid",
"type": "tuple"
},
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "signer", "type": "address" },
{
"internalType": "address",
"name": "collection",
"type": "address"
},
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{ "internalType": "uint256", "name": "amount", "type": "uint256" },
{ "internalType": "address", "name": "strategy", "type": "address" },
{ "internalType": "address", "name": "currency", "type": "address" },
{ "internalType": "uint256", "name": "nonce", "type": "uint256" },
{ "internalType": "uint256", "name": "startTime", "type": "uint256" },
{ "internalType": "uint256", "name": "endTime", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" },
{ "internalType": "uint8", "name": "v", "type": "uint8" },
{ "internalType": "bytes32", "name": "r", "type": "bytes32" },
{ "internalType": "bytes32", "name": "s", "type": "bytes32" }
],
"internalType": "struct OrderTypes.MakerOrder",
"name": "makerAsk",
"type": "tuple"
}
],
"name": "matchAskWithTakerBidUsingETHAndWETH",
"outputs": [],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "taker", "type": "address" },
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" }
],
"internalType": "struct OrderTypes.TakerOrder",
"name": "takerAsk",
"type": "tuple"
},
{
"components": [
{ "internalType": "bool", "name": "isOrderAsk", "type": "bool" },
{ "internalType": "address", "name": "signer", "type": "address" },
{
"internalType": "address",
"name": "collection",
"type": "address"
},
{ "internalType": "uint256", "name": "price", "type": "uint256" },
{ "internalType": "uint256", "name": "tokenId", "type": "uint256" },
{ "internalType": "uint256", "name": "amount", "type": "uint256" },
{ "internalType": "address", "name": "strategy", "type": "address" },
{ "internalType": "address", "name": "currency", "type": "address" },
{ "internalType": "uint256", "name": "nonce", "type": "uint256" },
{ "internalType": "uint256", "name": "startTime", "type": "uint256" },
{ "internalType": "uint256", "name": "endTime", "type": "uint256" },
{
"internalType": "uint256",
"name": "minPercentageToAsk",
"type": "uint256"
},
{ "internalType": "bytes", "name": "params", "type": "bytes" },
{ "internalType": "uint8", "name": "v", "type": "uint8" },
{ "internalType": "bytes32", "name": "r", "type": "bytes32" },
{ "internalType": "bytes32", "name": "s", "type": "bytes32" }
],
"internalType": "struct OrderTypes.MakerOrder",
"name": "makerBid",
"type": "tuple"
}
],
"name": "matchBidWithTakerAsk",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [],
"name": "owner",
"outputs": [{ "internalType": "address", "name": "", "type": "address" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "protocolFeeRecipient",
"outputs": [{ "internalType": "address", "name": "", "type": "address" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "renounceOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [],
"name": "royaltyFeeManager",
"outputs": [
{
"internalType": "contract IRoyaltyFeeManager",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{ "internalType": "address", "name": "newOwner", "type": "address" }
],
"name": "transferOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [],
"name": "transferSelectorNFT",
"outputs": [
{
"internalType": "contract ITransferSelectorNFT",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "_currencyManager",
"type": "address"
}
],
"name": "updateCurrencyManager",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "_executionManager",
"type": "address"
}
],
"name": "updateExecutionManager",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "_protocolFeeRecipient",
"type": "address"
}
],
"name": "updateProtocolFeeRecipient",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "_royaltyFeeManager",
"type": "address"
}
],
"name": "updateRoyaltyFeeManager",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "_transferSelectorNFT",
"type": "address"
}
],
"name": "updateTransferSelectorNFT",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [{ "internalType": "address", "name": "", "type": "address" }],
"name": "userMinOrderNonce",
"outputs": [{ "internalType": "uint256", "name": "", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
}
]

@ -0,0 +1,595 @@
[
{
"inputs": [],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
}
],
"name": "CollectionDisabled",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
}
],
"name": "CollectionUpdated",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "previousOwner",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "OwnershipTransferred",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "value",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "fromAddress",
"type": "address"
}
],
"name": "TokenBidEntered",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "value",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "fromAddress",
"type": "address"
}
],
"name": "TokenBidWithdrawn",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "value",
"type": "uint256"
},
{
"indexed": false,
"internalType": "address",
"name": "fromAddress",
"type": "address"
},
{
"indexed": false,
"internalType": "address",
"name": "toAddress",
"type": "address"
}
],
"name": "TokenBought",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "TokenNoLongerForSale",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"indexed": false,
"internalType": "uint256",
"name": "minValue",
"type": "uint256"
},
{
"indexed": true,
"internalType": "address",
"name": "toAddress",
"type": "address"
}
],
"name": "TokenOffered",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"internalType": "address",
"name": "collectionAddress",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "from",
"type": "address"
},
{
"indexed": true,
"internalType": "address",
"name": "to",
"type": "address"
},
{
"indexed": false,
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "TokenTransfer",
"type": "event"
},
{
"inputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"name": "collectionState",
"outputs": [
{
"internalType": "bool",
"name": "status",
"type": "bool"
},
{
"internalType": "bool",
"name": "erc1155",
"type": "bool"
},
{
"internalType": "uint256",
"name": "royaltyPercent",
"type": "uint256"
},
{
"internalType": "string",
"name": "metadataURL",
"type": "string"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "owner",
"outputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "",
"type": "address"
}
],
"name": "pendingBalance",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "renounceOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "",
"type": "address"
},
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"name": "tokenBids",
"outputs": [
{
"internalType": "bool",
"name": "hasBid",
"type": "bool"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"internalType": "address",
"name": "bidder",
"type": "address"
},
{
"internalType": "uint256",
"name": "value",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "",
"type": "address"
},
{
"internalType": "uint256",
"name": "",
"type": "uint256"
}
],
"name": "tokenOffers",
"outputs": [
{
"internalType": "bool",
"name": "isForSale",
"type": "bool"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"internalType": "address",
"name": "seller",
"type": "address"
},
{
"internalType": "uint256",
"name": "minValue",
"type": "uint256"
},
{
"internalType": "address",
"name": "onlySellTo",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "newOwner",
"type": "address"
}
],
"name": "transferOwnership",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "bool",
"name": "erc1155",
"type": "bool"
},
{
"internalType": "uint256",
"name": "royaltyPercent",
"type": "uint256"
},
{
"internalType": "string",
"name": "metadataURL",
"type": "string"
}
],
"name": "updateCollection",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
}
],
"name": "disableCollection",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"internalType": "uint256",
"name": "minSalePriceInWei",
"type": "uint256"
}
],
"name": "offerTokenForSale",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"internalType": "uint256",
"name": "minSalePriceInWei",
"type": "uint256"
},
{
"internalType": "address",
"name": "toAddress",
"type": "address"
}
],
"name": "offerTokenForSaleToAddress",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "tokenNoLongerForSale",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "enterBidForToken",
"outputs": [],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "withdrawBidForToken",
"outputs": [],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
}
],
"name": "acceptOfferForToken",
"outputs": [],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "contractAddress",
"type": "address"
},
{
"internalType": "uint256",
"name": "tokenIndex",
"type": "uint256"
},
{
"internalType": "uint256",
"name": "minPrice",
"type": "uint256"
}
],
"name": "acceptBidForToken",
"outputs": [],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [],
"name": "withdraw",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
}
]

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

5701
package-lock.json generated

File diff suppressed because it is too large Load Diff

@ -0,0 +1,29 @@
{
"name": "art101-sales-scraper",
"version": "1.0.0",
"engines": {
"node": "16.x"
},
"description": "",
"main": "src/main.js",
"scripts": {
"serve": "node --max_old_space_size=128 --optimize_for_size src/server.js",
"start": "concurrently npm:scrape npm:serve --restart-tries -1 --restart-after 5000",
"stop": "pkill -e -f concurrently && pkill -e -f scrape",
"scrape": "node --max_old_space_size=128 --optimize_for_size src/scraper.js",
"resync": "echo Deleting data in 5 seconds... && sleep 5 && rm storage/*.txt",
"wipe": "echo Deleting data in 5 seconds... && sleep 5 && npm run resync && storage/*.db"
},
"author": "lza_menace",
"license": "ISC",
"dependencies": {
"better-sqlite3": "^7.4.5",
"bignumber.js": "^9.0.1",
"concurrently": "^6.5.0",
"dotenv": "^10.0.0",
"ethers": "^5.6.9",
"express": "^4.17.1",
"node-fetch": "^3.2.9",
"sqlite3": "^5.0.2"
}
}

@ -0,0 +1,186 @@
<html>
<head>
<title>Art101 Sales Statistics</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
#container {
display: grid;
grid-template-columns: 5fr 2fr;
grid-gap: 2.5rem;
visibility: hidden;
}
#container > div {
align-self: center;
}
#loading {
font-weight: bold;
}
body {
font-family: "Open Sans";
color: black;
padding: 20px;
}
h1 {
margin-bottom: 0;
}
</style>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet">
</head>
<body>
<h1>Art101 Sales Statistics</h1>
<p><a href="https://github.com/lalanza808/nft-sales-scraper" target="_blank">Source Code</a></p>
<div id="loading">
Loading, please wait...
</div>
<div id="container"></div>
<script>
const container = document.getElementById('container');
fetch('/api/contracts').then(function(response) {
response.json().then(function(json) {
for (contract in json) {
console.log(`Found contract ${contract} (${json[contract]['contract_address']})`)
const contractName = contract;
const contractAddress = json[contract]['contract_address'];
const salesId = `sales-chart-${contractName}`;
const platformId = `platform-chart-${contractName}`;
newSalesChartDiv = document.createElement('div');
newSalesChart = document.createElement('canvas');
newSalesChart.setAttribute('id', salesId);
newSalesChartDiv.appendChild(newSalesChart);
newPlatformChartDiv = document.createElement('div');
newPlatformChart = document.createElement('canvas');
newPlatformChart.setAttribute('id', platformId);
newPlatformChartDiv.appendChild(newPlatformChart);
container.appendChild(newSalesChartDiv);
container.appendChild(newPlatformChartDiv);
let loaded = 0;
fetch(`/api/${contractAddress}/data`).then(function(response) {
response.json().then(function(json) {
loaded++;
if (loaded == 2) {
document.getElementById("container").style.visibility = "visible";
document.getElementById("loading").style.display = "none";
}
const labels = json.map(d => d.date);
const data = {
labels: labels,
datasets: [
{
type: 'line',
label: 'Average price',
backgroundColor: 'rgb(99, 132, 255)',
borderColor: 'rgb(99, 132, 255)',
data: json.map(d => d.average_price),
yAxisID: 'y1',
},
{
type: 'line',
label: 'Floor price',
backgroundColor: '#ccc',
borderColor: '#ccc',
data: json.map(d => d.floor_price),
yAxisID: 'y1',
},
{
type: 'bar',
label: 'Volume',
backgroundColor: 'rgb(255, 99, 132)',
borderColor: 'rgb(255, 99, 132)',
data: json.map(d => d.volume),
yAxisID: 'y',
}]
};
const config = {
type: 'line',
data: data,
options: {
elements: {
point: {
radius: 0
}
},
plugins: {
title: {
display: true,
text: `Sales for ${contractName}`,
font: {
size: '18px'
}
}
},
scales: {
y: {
type: 'linear',
display: true,
position: 'left',
},
y1: {
type: 'linear',
display: true,
position: 'right',
grid: {
drawOnChartArea: false,
},
},
}
}
};
const myChart = new Chart(
document.getElementById(salesId),
config
);
})
});
fetch(`/api/${contractAddress}/platforms`).then(function(response) {
response.json().then(function(json) {
loaded++;
if (loaded == 2) {
document.getElementById("container").style.visibility = "visible";
document.getElementById("loading").style.display = "none";
}
const labels = json.map(d => d.platform);
const total = json.map(d => d.volume).reduce((prev,next) => prev+next, 0);
const data = {
labels: labels,
datasets: [
{
label: 'Platform',
data: json.map(d => d.volume),
backgroundColor: ["#7463A8", "#80B7D8", "#AC39A0", "#B0C484", "#B75055", "#F12055"]
}]
};
const config = {
type: 'doughnut',
data: data,
options: {
plugins: {
title: {
display: true,
text: `Volume for ${contractName}: ${total.toFixed(2)}Ξ`,
font: {
size: '18px'
}
},
legend: {
position: 'bottom',
},
}
}
};
const myChart = new Chart(
document.getElementById(platformId),
config
);
})
});
}
})
})
</script>
</body>
</html>

@ -0,0 +1,58 @@
const fs = require('fs');
const { ethers } = require('ethers');
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
if (fs.existsSync('.env.local')) {
require('dotenv').config({path: '.env.local'});
} else {
console.warn('[!] No .env.local found, quitting.');
process.exit();
}
const assetsBase = 'https://art101-assets.s3.us-west-2.amazonaws.com';
async function postDiscord(_q) {
if (process.env.DISCORD_ACTIVE == 0) return
const contractAddress = ethers.utils.getAddress(_q.contractAddress);
try {
const title = `Sale of token ${_q.tokenId} for ${_q.contractName}!`;
const desc = `Purchased by ${shortenAddress(_q.targetOwner)} at <t:${Number(_q.txDate.getTime()) / 1000}> for ${ethers.utils.formatEther(_q.amount.toString())}Ξ on ${camelCase(_q.eventSource)}`;
const url = `${assetsBase}/${contractAddress}/${_q.tokenId.toString()}.json`;
const metadata = await fetch(url)
.then((r) => r.json());
const imageURL = metadata.image.replace('ipfs://', `${assetsBase}/${contractAddress}/`);
const res = await fetch(process.env.DISCORD_WEBHOOK, {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
embeds: [
{
title: title,
description: desc,
image: {
url: imageURL
},
url: `https://etherscan.io/tx/${_q.txHash}`
}
]
})
});
} catch(err) {
throw new Error(`[!] Failed to post to Discord: ${err}`);
}
}
function camelCase(s) {
return s.charAt(0).toUpperCase() + s.slice(1);
}
function shortenAddress(address) {
const shortAddress = `${address.slice(0, 6)}...${address.slice(address.length - 4, address.length)}`;
if (address.startsWith('0x')) return shortAddress;
return address;
}
module.exports = { postDiscord }

@ -0,0 +1,400 @@
const { BigNumber, ethers } = require('ethers');
const fs = require('fs');
const { Database } = require('sqlite3');
const { postDiscord } = require('./poster');
if (fs.existsSync('.env.local')) {
require('dotenv').config({path: '.env.local'});
} else {
console.warn('[!] No .env.local found, quitting.');
process.exit();
}
const CHUNK_SIZE = Number(process.env.CHUNK_SIZE);
const ALL_CONTRACTS = require('../data/contracts');
const ERC721_ABI = require('../data/erc721');
const ERC1155_ABI = require('../data/erc1155');
const MARKETPLACE_ABI = require('../data/marketplace');
const SEAPORT_ABI = require('../data/seaport');
const WYVERN_ABI = require('../data/wyvern');
const LOOKSRARE_ABI = require('../data/looksrare');
const WETH_ADDRESS = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2';
const TRANSFER_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef';
const TRANSFER_SINGLE_TOPIC = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62';
const LOOKSRARE_SALE_TOPIC = '0x95fb6205e23ff6bda16a2d1dba56b9ad7c783f67c96fa149785052f47696f2be';
const SEAPORT_SALE_TOPIC = '0x9d9af8e38d66c62e2c12f0225249fd9d721c54b83f48d9352c97c6cacdcb6f31';
const WYVERN_SALE_TOPIC = '0xc4109843e0b7d514e4c093114b863f8e7d8d9a458c372cd51bfe526b588006c9'
const X2Y2_SALE_TOPIC = '0x3cbb63f144840e5b1b0a38a7c19211d2e89de4d7c5faf8b2d3c1776c302d1d33';
const seaportInterface = new ethers.utils.Interface(SEAPORT_ABI);
const looksrareInterface = new ethers.utils.Interface(LOOKSRARE_ABI);
const wyvernInterface = new ethers.utils.Interface(WYVERN_ABI);
const provider = new ethers.providers.WebSocketProvider(process.env.GETH_NODE);
const db = new Database('./storage/sqlite.db');
class Collection {
constructor (contractName) {
if (!(contractName in ALL_CONTRACTS)) {
console.warn(`[!] That contract name does not exist in data/contracts.json`);
process.exit();
}
const data = ALL_CONTRACTS[contractName];
this.contractName = contractName;
this.contractAddress = data['contract_address'].toLowerCase();
this.erc1155 = data['erc1155'];
this.startBlock = data['start_block'];
if (this.erc1155) {
this.abi = ERC1155_ABI;
this.transferEvent = 'TransferSingle';
} else {
this.abi = ERC721_ABI;
this.transferEvent = 'Transfer';
}
this.interface = new ethers.utils.Interface(this.abi);
}
}
class Scrape extends Collection {
provider = this.getWeb3Provider();
constructor (contractName) {
super(contractName);
this.contract = new ethers.Contract(this.contractAddress, this.abi, this.provider);
this.lastFile = `./storage/lastBlock.${this.contractName}.txt`;
createDatabaseIfNeeded();
}
// ethereum chain provider - geth, infura, alchemy, etc
getWeb3Provider() {
return provider;
}
// continuous scanning loop
async scrape() {
let latestEthBlock = await this.provider.getBlockNumber();
let lastScrapedBlock = this.getLastBlock();
while (true) {
const lastRequested = lastScrapedBlock;
await this.filterTransfers(lastScrapedBlock).then(async ev => {
// capture transfer events with returned array of Transfers
try {
await this.getTransferEvents(ev);
} catch(err) {
console.log(ev)
throw new Error(err);
}
// filter down unique transaction hashes
ev.map(tx => tx.transactionHash).filter((tx, i, a) => a.indexOf(tx) === i).map(async txHash => {
// capture sales events for each
try {
await this.getSalesEvents(txHash);
} catch(err) {
console.log(txHash)
throw new Error(err);
}
});
});
if (lastRequested === lastScrapedBlock) {
lastScrapedBlock += CHUNK_SIZE;
this.writeLastBlock(lastScrapedBlock);
if (lastScrapedBlock > latestEthBlock) lastScrapedBlock = latestEthBlock;
}
while (lastScrapedBlock >= latestEthBlock) {
latestEthBlock = await this.provider.getBlockNumber();
console.log(`[ ${(new Date()).toISOString()} ][ ${this.contractName} ] [ waiting ]\n`)
await sleep(120);
}
}
}
// query historical logs
async filterTransfers(startBlock) {
let transfers;
console.log(`[ ${(new Date()).toISOString()} ][ ${this.contractName} ][ scraping ] blocks ${startBlock} - ${startBlock + CHUNK_SIZE}\n`);
if (this.erc1155) {
transfers = this.contract.filters.TransferSingle(null, null);
} else {
transfers = this.contract.filters.Transfer(null, null);
}
let res = await this.contract.queryFilter(transfers, startBlock, startBlock + CHUNK_SIZE)
return res;
}
// get transfer events from a batch from filtering
async getTransferEvents(txEvents) {
let platform = 'contract';
txEvents.forEach(async tx => {
let tokenId;
if (this.erc1155) {
tokenId = tx.args.id.toString();
} else {
tokenId = tx.args.tokenId.toString();
}
const fromAddress = tx.args.from.toString().toLowerCase();
const toAddress = tx.args.to.toString().toLowerCase();
const timestamp = await this.getBlockTimestamp(tx.blockNumber);
let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ transfer ] #${tokenId}: ${fromAddress} => ${toAddress} in tx ${tx.transactionHash}:${tx.logIndex}\n`;
console.log(msg);
const q = {
txHash: tx.transactionHash,
logIndex: tx.logIndex,
contractName: this.contractName,
contractAddress: this.contractAddress,
eventName: "transfer",
eventSource: "contract",
sourceOwner: fromAddress,
targetOwner: toAddress,
tokenId: tokenId,
amount: 0,
txDate: timestamp
}
writeToDatabase(q)
.then((res) => this.writeLastBlock(tx.blockNumber))
.catch((err) => console.log(`Error writing to database: ${err}`));
});
}
// get sales events from a given transaction
async getSalesEvents(txHash) {
try {
const receipt = await this.provider.getTransactionReceipt(txHash);
const timestamp = await this.getBlockTimestamp(receipt.blockNumber);
// Evaluate each log entry and determine if it's a sale for our contract and use custom logic for each exchange to parse values
receipt.logs.map(async log => {
let logIndex = log.logIndex;
let sale = false;
let platform;
let fromAddress;
let toAddress;
let amountWei;
let amountEther;
let tokenId;
if (log.topics[0].toLowerCase() === SEAPORT_SALE_TOPIC.toLowerCase()) {
// Handle Opensea/Seaport sales
const logDescription = seaportInterface.parseLog(log)
const matchingOffers = logDescription.args.offer.filter(
o => o.token.toLowerCase() == this.contractAddress
);
if (matchingOffers.length === 0) return;
sale = true;
platform = 'opensea';
fromAddress = logDescription.args.offerer.toLowerCase();
toAddress = logDescription.args.recipient.toLowerCase();
tokenId = logDescription.args.offer.map(o => o.identifier.toString());
let amounts = logDescription.args.consideration.map(c => BigInt(c.amount));
// add weth
const wethOffers = matchingOffers.map(o => o.token.toLowerCase() === WETH_ADDRESS.toLowerCase() && o.amount > 0 ? BigInt(o.amount) : BigInt(0));
if (wethOffers.length > 0 && wethOffers[0] != BigInt(0)) {
amounts = wethOffers
}
amountWei = amounts.reduce((previous,current) => previous + current, BigInt(0));
} else if (log.topics[0].toLowerCase() === WYVERN_SALE_TOPIC.toLowerCase()) {
// Handle Opensea/Wyvern sales
const logDescription = wyvernInterface.parseLog(log);
sale = true;
platform = 'opensea';
if (this.erc1155) {
const txLog = receipt.logs.map(l => l).filter(_l =>
(_l.topics[0].toLowerCase() == TRANSFER_SINGLE_TOPIC.toLowerCase())
).map(t => this.interface.parseLog(t))[0].args;
fromAddress = txLog.from.toLowerCase();
toAddress = txLog.to.toLowerCase();
tokenId = BigNumber.from(txLog.id).toString();
} else {
const txLog = receipt.logs.map(l => l).filter(_l =>
(_l.topics[0].toLowerCase() == TRANSFER_TOPIC.toLowerCase())
).map(t => this.interface.parseLog(t))[0].args;
fromAddress = txLog.from.toLowerCase();
toAddress = txLog.to.toLowerCase();
tokenId = BigNumber.from(txLog.tokenId).toString();
}
amountWei = BigInt(logDescription.args.price);
} else if (log.topics[0].toLowerCase() === LOOKSRARE_SALE_TOPIC.toLowerCase()) {
// Handle LooksRare sales
const logDescription = looksrareInterface.parseLog(log);
if (logDescription.args.collection.toLowerCase() != this.contractAddress) return;
sale = true;
platform = 'looksrare';
fromAddress = logDescription.args.maker.toLowerCase();
toAddress = receipt.from.toLowerCase();
tokenId = logDescription.args.tokenId.toString();
amountWei = logDescription.args.price.toString();
} else if (log.topics[0].toLowerCase() === X2Y2_SALE_TOPIC.toLowerCase()) {
// Handle x2y2 sales
const data = log.data.substring(2);
const dataSlices = data.match(/.{1,64}/g);
sale = true;
platform = 'x2y2';
fromAddress = BigNumber.from(`0x${dataSlices[0]}`)._hex.toString().toLowerCase();
toAddress = BigNumber.from(`0x${dataSlices[1]}`)._hex.toString().toLowerCase();
tokenId = BigInt(`0x${dataSlices[18]}`).toString();
amountWei = BigInt(`0x${dataSlices[12]}`);
if (amountWei === BigInt(0)) {
amountWei = BigInt(`0x${dataSlices[26]}`);
}
}
if (sale) {
amountEther = ethers.utils.formatEther(amountWei);
let msg = `[ ${timestamp.toISOString()} ][ ${this.contractName} ][ sale ] #${tokenId}: ${fromAddress} => ${toAddress} for ${amountEther}Ξ (${platform}) in tx ${txHash}:${logIndex}\n`;
console.log(msg);
const q = {
txHash: txHash,
logIndex: logIndex,
contractName: this.contractName,
contractAddress: this.contractAddress,
eventName: 'sale',
eventSource: platform,
sourceOwner: fromAddress,
targetOwner: toAddress,
tokenId: tokenId,
amount: amountWei,
txDate: timestamp
}
writeToDatabase(q)
.then((res) => this.writeLastBlock(log.blockNumber))
.catch((err) => console.log(`Error writing to database: ${err}`));
await postDiscord(q);
}
});
} catch(err) {
console.log(err);
}
}
/* Helpers */
// get stored block index to start scraping from
getLastBlock() {
let last = 0;
if (fs.existsSync(this.lastFile)) {
// read block stored in lastBlock file
last = parseInt(fs.readFileSync(this.lastFile).toString(), 10);
} else {
// write starting block if lastBlock file doesn't exist
fs.writeFileSync(this.lastFile, this.startBlock.toString());
};
// contract creation
if (Number.isNaN(last) || last < this.startBlock) {
last = this.startBlock;
};
return last;
}
// write last block to local filesystem
writeLastBlock(blockNumber) {
fs.writeFileSync(this.lastFile, blockNumber.toString());
}
// return date object of a given block
async getBlockTimestamp(blockNumber) {
const block = await this.provider.getBlock(blockNumber);
const d = new Date(block.timestamp * 1000);
return d;
}
}
async function sleep(sec) {
return new Promise((resolve) => setTimeout(resolve, Number(sec) * 1000));
}
async function createDatabaseIfNeeded() {
const tableExists = await new Promise((resolve) => {
db.get('SELECT name FROM sqlite_master WHERE type="table" AND name="events"', [], (err, row) => {
if (err) {
resolve(false);
}
resolve(row !== undefined);
});
});
if (!tableExists) {
db.serialize(() => {
db.run(
`CREATE TABLE events (
contract text, event_type text, from_wallet text, to_wallet text,
token_id number, amount number, tx_date text, tx text,
log_index number, platform text,
UNIQUE(tx, log_index)
);`,
);
db.run('CREATE INDEX idx_type_date ON events(event_type, tx_date);');
db.run('CREATE INDEX idx_date ON events(tx_date);');
db.run('CREATE INDEX idx_amount ON events(amount);');
db.run('CREATE INDEX idx_platform ON events(platform);');
db.run('CREATE INDEX idx_contract ON events(contract);');
db.run('CREATE INDEX idx_tx ON events(tx);');
});
}
}
async function checkRowExists(txHash, logIndex) {
const rowExists = await new Promise((resolve) => {
db.get('SELECT * FROM events WHERE tx = ? AND log_index = ?', [txHash, logIndex], (err, row) => {
if (err) {
resolve(false);
}
resolve(row !== undefined);
});
});
return rowExists;
}
async function writeToDatabase(_q) {
// txHash, logIndex, contractName, contractAddress, eventName, eventSource, sourceOwner, targetOwner, tokenId, amount, txDate
const rowExists = await checkRowExists(_q.txHash, _q.logIndex, _q.contractAddress);
if (!rowExists) {
let stmt;
try {
stmt = db.prepare('INSERT INTO events VALUES (?,?,?,?,?,?,?,?,?,?)');
stmt.run(
_q.contractAddress,
_q.eventName,
_q.sourceOwner,
_q.targetOwner,
_q.tokenId,
_q.amount.toString(),
_q.txDate.toISOString(),
_q.txHash,
_q.logIndex,
_q.eventSource
);
stmt.finalize();
return true;
} catch(err) {
console.log(`Error when writing to database: ${err}`);
console.log(`Query: ${stmt}`);
return false;
}
}
return true;
}
// Sample events for testing functionality and detecting sales
// let c = new Scrape('non-fungible-soup');
// c.getSalesEvents('0x2f8961209daca23288c499449aa936b54eec5c25720b9d7499a8ee5bde7fcdc7')
// c.getSalesEvents('0xb20853f22b367ee139fd800206bf1cba0c36f1a1dd739630f99cc6ffd0471edc')
// c.getSalesEvents('0x71e5135a543e17cc91992a2229ae5811461c96b84d5e2560ac8db1dd99bb17e3')
// c.getSalesEvents('0x5dc68e0bd60fa671e7b6702002e4ce374de6a5dd49fcda00fdb45e26771bcbd9')
// c.getSalesEvents('0x975d10cdd873ee5bb29e746c2f1f3b776078cace9c04ce419cb66949239288b5')
// c.getSalesEvents('0x8d45ed8168a740f8b182ec0dbad1c37d6c6dbd8aa865be408d865ca01fb0fa94')
// c.getSalesEvents('0x27ab6f12604bf17a9e7c93bf1a7cc466d7dfd922565d267eac10879b59d5d0b5')
// c.getSalesEvents('0x511bc5cda2b7145511c7b57e29cecf1f15a5a650670f09e91e69fc24824effd9')
// c.getSalesEvents('0x04746b6ba1269906db8e0932263b86a6fc35a30a31cf73d2b7db078f6f4ed442')
// c.getSalesEvents('0x24d6523c5048b2df3e7f8b24d63a6644e4c0ed33cfae6396190e3ded5fc79321')
// c.getSalesEvents('0xe56dc64c44a3cbfe3a1e68f8669a65f17ebe48d64e944673122a565b7c641d1e')
// return
if (process.env.SCRAPE) {
let c = new Scrape(process.env.SCRAPE)
c.scrape()
} else {
for(const key in ALL_CONTRACTS) {
const c = new Scrape(key);
c.scrape();
}
}

@ -0,0 +1,165 @@
const express = require('express');
const Database = require('better-sqlite3');
const fs = require('fs');
if (fs.existsSync('.env.local')) {
require('dotenv').config({path: '.env.local'});
} else {
console.warn('[!] No .env.local found, quitting.');
process.exit();
}
const ALL_CONTRACTS = require('../data/contracts');
const db = new Database('./storage/sqlite.db');
const app = express();
const port = process.env.PORT || 3000;
app.use(express.json());
app.use('/', express.static('public'));
app.use('/app', express.static('public'));
app.get('/api/contracts', (req, res) => {
res.status(200).json(ALL_CONTRACTS)
})
app.get('/api/:contractAddress/offers', (req, res) => {
const results = [];
const stmt = db.prepare(`select *
from events ev
where contract = '${req.params.contractAddress}'
collate nocase
and event_type = 'tokenoffered'
and not token_id in
(
select token_id from events
where event_type == 'tokennolongerforsale'
and token_id = token_id
and contract = '${req.params.contractAddress}'
collate nocase
and tx_date > ev.tx_date
order by tx_date asc
limit 1
)
order by tx_date desc
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.get('/api/:contractAddress/bids', (req, res) => {
const results = [];
const stmt = db.prepare(`select *
from events ev
where contract = '${req.params.contractAddress}'
collate nocase
and event_type = 'tokenbidentered'
and not token_id in
(
select token_id from events
where event_type == 'tokenbidwithdrawn'
and token_id = token_id
and contract = '${req.params.contractAddress}'
collate nocase
and tx_date > ev.tx_date
order by tx_date asc
limit 1
)
order by tx_date desc
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.get('/api/:contractAddress/events', (req, res) => {
const results = [];
const stmt = db.prepare(`select *
from events
where contract = '${req.params.contractAddress}'
collate nocase
and event_type != 'sale' and event_type != 'transfer'
order by tx_date desc
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.get('/api/token/:contractAddress/:tokenId/history', (req, res) => {
const results = [];
const stmt = db.prepare(`select *
from events
where token_id = ${req.params.tokenId}
and contract = '${req.params.contractAddress}'
collate nocase
order by tx_date desc
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.get('/api/latest', (req, res) => {
const stmt = db.prepare(`select *
from events
order by tx_date desc
limit 1
`);
res.status(200).json(stmt.get());
});
app.get('/api/:contractAddress/data', (req, res) => {
const results = [];
const stmt = db.prepare(`select
date(tx_date) date,
sum(amount/1000000000000000000.0) volume,
avg(amount/1000000000000000000.0) average_price,
(select avg(amount/1000000000000000000.0) from (select * from events
where event_type == 'sale'
and contract = '${req.params.contractAddress}'
collate nocase
and date(tx_date) = date(ev.tx_date)
order by amount
limit 10)) floor_price,
count(*) sales
from events ev
where event_type == 'sale'
and contract = '${req.params.contractAddress}'
collate nocase
group by date(tx_date)
order by date(tx_date)
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.get('/api/:contractAddress/platforms', (req, res) => {
const results = [];
const stmt = db.prepare(`select platform,
sum(amount/1000000000000000000.0) volume,
count(*) sales
from events
where event_type = 'sale'
and contract = '${req.params.contractAddress}'
collate nocase
group by platform
order by sum(amount/1000000000000000000.0) desc
`);
for (const entry of stmt.iterate()) {
results.push(entry);
}
res.status(200).json(results);
});
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`);
});
Loading…
Cancel
Save