moonstream/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py

import argparse
import logging
from typing import Optional

from moonstreamdb.db import yield_db_session_ctx
from web3 import Web3
from web3.middleware import geth_poa_middleware
import json
from mooncrawl.data import AvailableBlockchainType  # type: ignore
from ..blockchain import connect
from .base import crawl, get_checkpoint, populate_with_events

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def handle_nft_crawler(args: argparse.Namespace) -> None:
    logger.info(f"Starting NFT crawler")

    with open("mooncrawl/generic_crawler/abis/erc721.json") as f:
        abi = json.load(f)

    label = args.label_name
    from_block = args.start_block
    to_block = args.end_block

    # january_2021_block = 9013700  # for polygon
    # to_block = 24975113  # for polygon, 15 february 2022

    blockchain_type = AvailableBlockchainType(args.blockchain_type)

    logger.info(f"Blockchain type: {blockchain_type.value}")
    with yield_db_session_ctx() as db_session:
        web3: Optional[Web3] = None
        if args.web3 is None:
            logger.info(
                "No web3 provider URL provided, using default (blockchan.py: connect())"
            )
            web3 = connect(blockchain_type)
        else:
            logger.info(f"Using web3 provider URL: {args.web3}")
            web3 = Web3(
                Web3.HTTPProvider(args.web3),
            )
            if args.poa:
                logger.info("Using PoA middleware")
                web3.middleware_onion.inject(geth_poa_middleware, layer=0)
        last_crawled_block = get_checkpoint(
            db_session, blockchain_type, from_block, to_block, label
        )

        logger.info(f"Starting from block: {last_crawled_block}")
        crawl(
            db_session,
            web3,
            blockchain_type,
            label,
            abi,
            [],
            from_block=last_crawled_block,
            to_block=to_block,
            batch_size=args.max_blocks_batch,
        )


def populate_with_erc20_transfers(args: argparse.Namespace) -> None:
    logger.info(f"Starting erc20 transfer crawler")

    label = args.label_name
    from_block = args.start_block
    to_block = args.end_block

    with open(args.abi) as f:
        erc20_abi = json.load(f)
        # Taking only transfer event from erc20_abi
        erc20_abi = [
            event
            for event in erc20_abi
            if event["type"] == "event" and event["name"] == "Transfer"
        ]

    blockchain_type = AvailableBlockchainType(args.blockchain_type)

    logger.info(f"Blockchain type: {blockchain_type.value}")
    with yield_db_session_ctx() as db_session:
        web3: Optional[Web3] = None
        if args.web3 is None:
            logger.info(
                "No web3 provider URL provided, using default (blockchan.py: connect())"
            )
            web3 = connect(blockchain_type)
        else:
            logger.info(f"Using web3 provider URL: {args.web3}")
            web3 = Web3(
                Web3.HTTPProvider(args.web3),
            )
            if args.poa:
                logger.info("Using PoA middleware")
                web3.middleware_onion.inject(geth_poa_middleware, layer=0)
        last_crawled_block = get_checkpoint(
            db_session, blockchain_type, from_block, to_block, label
        )

        logger.info(f"Starting from block: {last_crawled_block}")
        populate_with_events(
            db_session,
            web3,
            blockchain_type,
            label,
            args.label_to_populate,
            erc20_abi,
            last_crawled_block,
            to_block,
            batch_size=args.max_blocks_batch,
        )


def handle_crawl(args: argparse.Namespace) -> None:
    logger.info(f"Starting generic crawler")

    label = args.label_name
    from_block = args.start_block
    to_block = args.end_block

    with open(args.abi) as f:
        abi = json.load(f)

    blockchain_type = AvailableBlockchainType(args.blockchain_type)

    logger.info(f"Blockchain type: {blockchain_type.value}")
    with yield_db_session_ctx() as db_session:
        web3: Optional[Web3] = None
        if args.web3 is None:
            logger.info(
                "No web3 provider URL provided, using default (blockchan.py: connect())"
            )
            web3 = connect(blockchain_type)
        else:
            logger.info(f"Using web3 provider URL: {args.web3}")
            web3 = Web3(
                Web3.HTTPProvider(args.web3),
            )
            if args.poa:
                logger.info("Using PoA middleware")
                web3.middleware_onion.inject(geth_poa_middleware, layer=0)
        last_crawled_block = get_checkpoint(
            db_session, blockchain_type, from_block, to_block, label
        )

        logger.info(f"Starting from block: {last_crawled_block}")
        crawl_transaction = not args.disable_transactions
        crawl(
            db_session,
            web3,
            blockchain_type,
            label,
            abi,
            secondary_abi=[],
            from_block=last_crawled_block,
            to_block=to_block,
            crawl_transactions=crawl_transaction,
            batch_size=args.max_blocks_batch,
        )


def main():
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi")
    crawl_parser.add_argument(
        "--blockchain_type",
        type=str,
        required=True,
        help=f"Available blockchain types: {[member.value for member in AvailableBlockchainType]}",
    )
    crawl_parser.add_argument(
        "--abi",
        type=str,
        default=None,
        help="Abi of the contract",
    )
    crawl_parser.add_argument(
        "--disable_transactions",
        action="store_true",
        help="Disable transactions crawling",
    )
    crawl_parser.add_argument(
        "--web3",
        type=str,
        default=None,
        help="Web3 provider URL",
    )

    crawl_parser.add_argument(
        "--poa",
        action="store_true",
        default=False,
        help="Use PoA middleware",
    )

    crawl_parser.add_argument(
        "--start_block",
        type=int,
        default=None,
    )
    crawl_parser.add_argument(
        "--end_block",
        type=int,
        default=None,
    )

    crawl_parser.add_argument(
        "--max_blocks_batch",
        type=int,
        default=500,
        help="Maximum number of blocks to crawl in a single crawl step",
    )

    crawl_parser.add_argument(
        "--label_name",
        type=str,
        default="erc721",
        help="Label name",
    )

    crawl_parser.set_defaults(func=handle_crawl)

    nft_crawler_parser = subparsers.add_parser(
        "nft",
        help="Run the NFT crawler",
    )

    nft_crawler_parser.add_argument(
        "--blockchain_type",
        type=str,
        required=True,
        choices=[
            "ethereum",
            "polygon",
        ],
    )
    nft_crawler_parser.add_argument(
        "--web3",
        type=str,
        default=None,
        help="Web3 provider URL",
    )

    nft_crawler_parser.add_argument(
        "--poa",
        action="store_true",
        default=False,
        help="Use PoA middleware",
    )

    nft_crawler_parser.add_argument(
        "--start_block",
        type=int,
        default=None,
    )
    nft_crawler_parser.add_argument(
        "--end_block",
        type=int,
        default=None,
    )

    nft_crawler_parser.add_argument(
        "--max_blocks_batch",
        type=int,
        default=500,
        help="Maximum number of blocks to crawl in a single crawl step",
    )

    nft_crawler_parser.add_argument(
        "--label_name",
        type=str,
        default="erc721",
        help="Label name",
    )

    nft_crawler_parser.set_defaults(func=handle_nft_crawler)

    erc20_populate_parser = subparsers.add_parser(
        "erc20_populate",
        help="Populate erc20 labels",
    )

    erc20_populate_parser.add_argument(
        "--blockchain_type",
        type=str,
        required=True,
        choices=[
            "ethereum",
            "polygon",
        ],
    )
    erc20_populate_parser.add_argument(
        "--web3",
        type=str,
        default=None,
        help="Web3 provider URL",
    )

    erc20_populate_parser.add_argument(
        "--poa",
        action="store_true",
        default=False,
        help="Use PoA middleware",
    )

    erc20_populate_parser.add_argument(
        "--start_block",
        type=int,
        default=None,
    )
    erc20_populate_parser.add_argument(
        "--end_block",
        type=int,
        default=None,
    )

    erc20_populate_parser.add_argument(
        "--max_blocks_batch",
        type=int,
        default=500,
        help="Maximum number of blocks to crawl in a single crawl step",
    )

    erc20_populate_parser.add_argument(
        "--label_name",
        type=str,
        required=True,
        help="Label name",
    )

    erc20_populate_parser.add_argument(
        "--label_to_populate",
        type=str,
        required=True,
        help="Label name to populate",
    )

    erc20_populate_parser.add_argument(
        "--abi",
        type=str,
        default=None,
        help="Abi of the erc20 contract",
    )

    erc20_populate_parser.set_defaults(func=populate_with_erc20_transfers)

    args = parser.parse_args()
    args.func(args)


if __name__ == "__main__":
    main()