From 5c8d0030796bd846a34358107328f962a8a80ad0 Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Fri, 25 Mar 2022 18:02:10 +0300 Subject: [PATCH] added tokenAddress --- datasets/nfts/nfts/cli.py | 9 +++++---- datasets/nfts/nfts/data.py | 4 ++++ datasets/nfts/nfts/datastore.py | 20 ++++++++++++++++---- datasets/nfts/nfts/materialize.py | 9 ++++++++- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/datasets/nfts/nfts/cli.py b/datasets/nfts/nfts/cli.py index 25e6e6ea..1730949a 100644 --- a/datasets/nfts/nfts/cli.py +++ b/datasets/nfts/nfts/cli.py @@ -55,11 +55,12 @@ def handle_initdb(args: argparse.Namespace) -> None: def handle_materialize(args: argparse.Namespace) -> None: + if args.start is None or args.end is None: + raise ValueError("Set --end --start") + bounds: Optional[BlockBounds] = None if args.start is not None: bounds = BlockBounds(starting_block=args.start, ending_block=args.end) - elif args.end is not None: - raise ValueError("You cannot set --end unless you also set --start") logger.info(f"Materializing NFT events to datastore: {args.datastore}") logger.info(f"Block bounds: {bounds}") @@ -78,8 +79,8 @@ def handle_materialize(args: argparse.Namespace) -> None: moonstream_datastore, label_model, start_block=bounds.starting_block, - end_block=bounds.starting_block + 500000, - batch_size=10000, + end_block=bounds.ending_block, + batch_size=args.batch_size, ) diff --git a/datasets/nfts/nfts/data.py b/datasets/nfts/nfts/data.py index 5190e62f..9a5debea 100644 --- a/datasets/nfts/nfts/data.py +++ b/datasets/nfts/nfts/data.py @@ -34,6 +34,7 @@ class NftTransaction: @dataclass class NftApprovalEvent: blockchain_type: str + token_address: str owner: str approved: str token_id: str @@ -44,6 +45,7 @@ class NftApprovalEvent: @dataclass class NftApprovalForAllEvent: blockchain_type: str + token_address: str owner: str approved: str operator: str @@ -54,6 +56,7 @@ class NftApprovalForAllEvent: @dataclass class NftTransferEvent: blockchain_type: str + token_address: str from_address: str to_address: str token_id: str @@ -64,6 +67,7 @@ class NftTransferEvent: @dataclass class Erc20TransferEvent: blockchain_type: str + token_address: str from_address: str to_address: str value: int diff --git a/datasets/nfts/nfts/datastore.py b/datasets/nfts/nfts/datastore.py index 06872af7..156a8b43 100644 --- a/datasets/nfts/nfts/datastore.py +++ b/datasets/nfts/nfts/datastore.py @@ -49,6 +49,7 @@ def create_approvals_table_query(tabel_name) -> str: CREATE TABLE IF NOT EXISTS {tabel_name} ( blockchainType TEXT NOT NULL, + tokenAddress TEXT NOT NULL, owner TEXT NOT NULL, approved TEXT NOT NULL, tokenId TEXT NOT NULL, @@ -65,6 +66,7 @@ def create_approval_for_all_table_query(tabel_name) -> str: CREATE TABLE IF NOT EXISTS {tabel_name} ( blockchainType TEXT NOT NULL, + tokenAddress TEXT NOT NULL, owner TEXT NOT NULL, approved BOOL NOT NULL, operator TEXT NOT NULL, @@ -81,6 +83,7 @@ def create_transfers_table_query(tabel_name) -> str: CREATE TABLE IF NOT EXISTS {tabel_name} ( blockchainType TEXT NOT NULL, + tokenAddress TEXT NOT NULL, from_address TEXT NOT NULL, to_address TEXT NOT NULL, tokenId TEXT NOT NULL, @@ -97,6 +100,7 @@ def create_erc20_transfers_table_query(tabel_name) -> str: CREATE TABLE IF NOT EXISTS {tabel_name} ( blockchainType TEXT NOT NULL, + tokenAddress TEXT NOT NULL, from_address TEXT NOT NULL, to_address TEXT NOT NULL, value INTEGER NOT NULL, @@ -139,6 +143,7 @@ def insert_nft_approval_query(tabel_name): INSERT INTO {tabel_name} ( blockchainType, + tokenAddress, owner, approved, tokenId, @@ -147,7 +152,7 @@ INSERT INTO {tabel_name} ) VALUES ( - ?,?,?,?,?,? + ?,?,?,?,?,?,? ); """ return query @@ -158,6 +163,7 @@ def insert_nft_approval_for_all_query(tabel_name): INSERT INTO {tabel_name} ( blockchainType, + tokenAddress, owner, approved, operator, @@ -166,7 +172,7 @@ INSERT INTO {tabel_name} ) VALUES ( - ?,?,?,?,?,? + ?,?,?,?,?,?, ? ); """ return query @@ -177,6 +183,7 @@ def insert_nft_transfers_query(tabel_name): INSERT INTO {tabel_name} ( blockchainType, + tokenAddress, from_address, to_address, tokenId, @@ -186,7 +193,7 @@ INSERT INTO {tabel_name} VALUES ( - ?,?,?,?,?,? + ?,?,?,?,?,?,? ); """ return query @@ -197,6 +204,7 @@ def insert_erc20_transfer_query(tabel_name): INSERT INTO {tabel_name} ( blockchainType, + tokenAddress, from_address, to_address, value, @@ -205,7 +213,7 @@ INSERT INTO {tabel_name} ) VALUES ( - ?,?,?,?,?,? + ?,?,?,?,?,?,? ); """ return query @@ -245,6 +253,7 @@ def nft_approval_to_tuple(nft_approval: NftApprovalEvent) -> Tuple[Any]: """ return ( nft_approval.blockchain_type, + nft_approval.token_address, nft_approval.owner, nft_approval.approved, str(nft_approval.token_id), @@ -261,6 +270,7 @@ def nft_approval_for_all_to_tuple( """ return ( nft_approval_for_all.blockchain_type, + nft_approval_for_all.token_address, nft_approval_for_all.owner, nft_approval_for_all.approved, nft_approval_for_all.operator, @@ -275,6 +285,7 @@ def nft_transfer_to_tuple(nft_transfer: NftTransferEvent) -> Tuple[Any]: """ return ( nft_transfer.blockchain_type, + nft_transfer.token_address, nft_transfer.from_address, nft_transfer.to_address, str(nft_transfer.token_id), @@ -291,6 +302,7 @@ def erc20_nft_transfer_to_tuple( """ return ( erc20_nft_transfer.blockchain_type, + erc20_nft_transfer.token_address, erc20_nft_transfer.from_address, erc20_nft_transfer.to_address, str(erc20_nft_transfer.value), diff --git a/datasets/nfts/nfts/materialize.py b/datasets/nfts/nfts/materialize.py index 852b48d4..29e1e59e 100644 --- a/datasets/nfts/nfts/materialize.py +++ b/datasets/nfts/nfts/materialize.py @@ -27,6 +27,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) ERC721_LABEL = "erc721" +ERC20_LABEL = "test-erc20" def _get_last_labeled_erc721_block( @@ -95,6 +96,7 @@ def _parse_transfer_event( if label_model.label_data["args"].get("tokenId") is not None: return NftTransferEvent( blockchain_type=blockchain_type, + token_address=label_model.address, from_address=label_model.label_data["args"]["from"], to_address=label_model.label_data["args"]["to"], token_id=label_model.label_data["args"]["tokenId"], @@ -104,6 +106,7 @@ def _parse_transfer_event( else: return Erc20TransferEvent( blockchain_type=blockchain_type, + token_address=label_model.address, from_address=label_model.label_data["args"]["from"], to_address=label_model.label_data["args"]["to"], value=label_model.label_data["args"]["value"], @@ -128,6 +131,7 @@ def _parse_approval_event( blockchain_type = "polygon" return NftApprovalEvent( blockchain_type=blockchain_type, + token_address=label_model.address, owner=label_model.label_data["args"]["owner"], approved=label_model.label_data["args"]["approved"], token_id=label_model.label_data["args"]["tokenId"], @@ -152,6 +156,7 @@ def _parse_approval_for_all_event( blockchain_type = "polygon" return NftApprovalForAllEvent( blockchain_type=blockchain_type, + token_address=label_model.address, owner=label_model.label_data["args"]["owner"], operator=label_model.label_data["args"]["operator"], approved=label_model.label_data["args"]["approved"], @@ -196,7 +201,9 @@ def crawl_erc721_labels( and_( label_model.block_number >= current_block, label_model.block_number <= batch_end, - label_model.label == ERC721_LABEL, + or_( + label_model.label == ERC721_LABEL, label_model.label == ERC20_LABEL + ), ) )