diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 2f2bd95a..1551deb7 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -22,6 +22,7 @@ from .data import DateRange from .db import yield_db_session, yield_db_session_ctx from .settings import ( MOONSTREAM_CRAWL_WORKERS, + MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI, MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI, MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI, MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI, @@ -66,6 +67,8 @@ def connect( web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.MUMBAI: web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.AMOY: + web3_uri = MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.XDAI: web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index ce024f2f..b4b1b424 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -4,6 +4,7 @@ from typing import Optional from uuid import UUID from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.subscriptions import blockchain_type_to_subscription_type from web3 import Web3 from web3.middleware import geth_poa_middleware @@ -15,8 +16,6 @@ from ..settings import ( ) from .continuous_crawler import _retry_connect_web3, continuous_crawler from .crawler import ( - SubscriptionTypes, - blockchain_type_to_subscription_type, find_all_deployed_blocks, get_crawl_job_entries, make_event_crawl_jobs, diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 990200a8..c23a8602 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple from uuid import UUID from moonstreamdb.blockchain import AvailableBlockchainType -from moonstreamdb.networks import Network +from moonstreamdb.networks import blockchain_type_to_network_type +from moonstreamdb.subscriptions import blockchain_type_to_subscription_type from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) @@ -21,7 +22,6 @@ from .crawler import ( EventCrawlJob, FunctionCallCrawlJob, _retry_connect_web3, - blockchain_type_to_subscription_type, get_crawl_job_entries, heartbeat, make_event_crawl_jobs, @@ -124,40 +124,10 @@ def continuous_crawler( if web3 is None: web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri) - if blockchain_type == AvailableBlockchainType.ETHEREUM: - network = Network.ethereum - elif blockchain_type == AvailableBlockchainType.POLYGON: - network = Network.polygon - elif blockchain_type == AvailableBlockchainType.MUMBAI: - network = Network.mumbai - elif blockchain_type == AvailableBlockchainType.XDAI: - network = Network.xdai - elif blockchain_type == AvailableBlockchainType.WYRM: - network = Network.wyrm - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - network = Network.zksync_era_testnet - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: - network = Network.zksync_era - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: - network = Network.zksync_era_sepolia - elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - network = Network.arbitrum_nova - elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - network = Network.arbitrum_sepolia - elif blockchain_type == AvailableBlockchainType.XAI: - network = Network.xai - elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: - network = Network.xai_sepolia - elif blockchain_type == AvailableBlockchainType.AVALANCHE: - network = Network.avalanche - elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: - network = Network.avalanche_fuji - elif blockchain_type == AvailableBlockchainType.BLAST: - network = Network.blast - elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: - network = Network.blast_sepolia - else: - raise ValueError(f"Unknown blockchain type: {blockchain_type}") + try: + network = blockchain_type_to_network_type(blockchain_type=blockchain_type) + except Exception as e: + raise Exception(e) ethereum_state_provider = MoonstreamEthereumStateProvider( web3, diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index e90910f1..62ad4c65 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -11,6 +11,7 @@ from uuid import UUID from bugout.data import BugoutJournalEntries, BugoutSearchResult from eth_typing.evm import ChecksumAddress from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.subscriptions import SubscriptionTypes from moonworm.deployment import find_deployment_block # type: ignore from web3.main import Web3 @@ -29,25 +30,6 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -class SubscriptionTypes(Enum): - POLYGON_BLOCKCHAIN = "polygon_smartcontract" - ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract" - MUMBAI_BLOCKCHAIN = "mumbai_smartcontract" - XDAI_BLOCKCHAIN = "xdai_smartcontract" - WYRM_BLOCKCHAIN = "wyrm_smartcontract" - ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract" - ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract" - ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN = "zksync_era_sepolia_smartcontract" - ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract" - ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract" - XAI_BLOCKCHAIN = "xai_smartcontract" - XAI_SEPOLIA_BLOCKCHAIN = "xai_sepolia_smartcontract" - AVALANCHE_BLOCKCHAIN = "avalanche_smartcontract" - AVALANCHE_FUJI_BLOCKCHAIN = "avalanche_fuji_smartcontract" - BLAST_BLOCKCHAIN = "blast_smartcontract" - BLAST_SEPOLIA_BLOCKCHAIN = "blast_sepolia_smartcontract" - - def abi_input_signature(input_abi: Dict[str, Any]) -> str: """ Stringifies a function ABI input object according to the ABI specification: @@ -137,45 +119,6 @@ def _retry_connect_web3( ) -def blockchain_type_to_subscription_type( - blockchain_type: AvailableBlockchainType, -) -> SubscriptionTypes: - if blockchain_type == AvailableBlockchainType.ETHEREUM: - return SubscriptionTypes.ETHEREUM_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.POLYGON: - return SubscriptionTypes.POLYGON_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.MUMBAI: - return SubscriptionTypes.MUMBAI_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.XDAI: - return SubscriptionTypes.XDAI_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.WYRM: - return SubscriptionTypes.WYRM_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: - return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: - return SubscriptionTypes.ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.XAI: - return SubscriptionTypes.XAI_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: - return SubscriptionTypes.XAI_SEPOLIA_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.AVALANCHE: - return SubscriptionTypes.AVALANCHE_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: - return SubscriptionTypes.AVALANCHE_FUJI_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.BLAST: - return SubscriptionTypes.BLAST_BLOCKCHAIN - elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: - return SubscriptionTypes.BLAST_SEPOLIA_BLOCKCHAIN - else: - raise ValueError(f"Unknown blockchain type: {blockchain_type}") - - @dataclass class EventCrawlJob: event_abi_hash: str diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 44834fdf..e3631790 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -2,7 +2,7 @@ import logging from typing import List from moonstreamdb.blockchain import AvailableBlockchainType -from moonstreamdb.networks import Network # type: ignore +from moonstreamdb.networks import blockchain_type_to_network_type # type: ignore from moonworm.crawler.function_call_crawler import ( # type: ignore ContractFunctionCall, FunctionCallCrawler, @@ -58,40 +58,10 @@ def function_call_crawler( end_block: int, batch_size: int, ): - if blockchain_type == AvailableBlockchainType.ETHEREUM: - network = Network.ethereum - elif blockchain_type == AvailableBlockchainType.POLYGON: - network = Network.polygon - elif blockchain_type == AvailableBlockchainType.MUMBAI: - network = Network.mumbai - elif blockchain_type == AvailableBlockchainType.XDAI: - network = Network.xdai - elif blockchain_type == AvailableBlockchainType.WYRM: - network = Network.wyrm - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - network = Network.zksync_era_testnet - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: - network = Network.zksync_era - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: - network = Network.zksync_era_sepolia - elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - network = Network.arbitrum_nova - elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - network = Network.arbitrum_sepolia - elif blockchain_type == AvailableBlockchainType.XAI: - network = Network.xai - elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: - network = Network.xai_sepolia - elif blockchain_type == AvailableBlockchainType.AVALANCHE: - network = Network.avalanche - elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: - network = Network.avalanche_fuji - elif blockchain_type == AvailableBlockchainType.BLAST: - network = Network.blast - elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: - network = Network.blast_sepolia - else: - raise ValueError(f"Unknown blockchain type: {blockchain_type}") + try: + network = blockchain_type_to_network_type(blockchain_type=blockchain_type) + except Exception as e: + raise Exception(e) ethereum_state_provider = MoonstreamEthereumStateProvider( web3, diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 6761616f..028f9e71 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -5,7 +5,7 @@ from uuid import UUID from eth_typing.evm import ChecksumAddress from moonstreamdb.blockchain import AvailableBlockchainType -from moonstreamdb.networks import Network # type: ignore +from moonstreamdb.networks import blockchain_type_to_network_type # type: ignore from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) @@ -52,40 +52,10 @@ def historical_crawler( web3.eth.block_number >= start_block ), "start_block must be less than current block" - if blockchain_type == AvailableBlockchainType.ETHEREUM: - network = Network.ethereum - elif blockchain_type == AvailableBlockchainType.POLYGON: - network = Network.polygon - elif blockchain_type == AvailableBlockchainType.MUMBAI: - network = Network.mumbai - elif blockchain_type == AvailableBlockchainType.XDAI: - network = Network.xdai - elif blockchain_type == AvailableBlockchainType.WYRM: - network = Network.wyrm - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - network = Network.zksync_era_testnet - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: - network = Network.zksync_era - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: - network = Network.zksync_era_sepolia - elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - network = Network.arbitrum_nova - elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - network = Network.arbitrum_sepolia - elif blockchain_type == AvailableBlockchainType.XAI: - network = Network.xai - elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: - network = Network.xai_sepolia - elif blockchain_type == AvailableBlockchainType.AVALANCHE: - network = Network.avalanche - elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: - network = Network.avalanche_fuji - elif blockchain_type == AvailableBlockchainType.BLAST: - network = Network.blast - elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: - network = Network.blast_sepolia - else: - raise Exception("Unsupported blockchain type provided") + try: + network = blockchain_type_to_network_type(blockchain_type=blockchain_type) + except Exception as e: + raise Exception(e) ethereum_state_provider = MoonstreamEthereumStateProvider( web3, diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 99d604ff..35fe1764 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -96,6 +96,12 @@ MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get( if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "": raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set") +MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI env variable is not set") + MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get( "MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", "" ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 74ca3375..b8c66e62 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -24,6 +24,10 @@ from moonstreamdb.blockchain import ( get_label_model, get_transaction_model, ) +from moonstreamdb.subscriptions import ( + blockchain_by_subscription_id, + subscription_id_by_blockchain, +) from sqlalchemy import and_ from sqlalchemy import cast as sqlalchemy_cast from sqlalchemy import distinct, extract, func, text @@ -47,61 +51,6 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -subscription_id_by_blockchain = { - "ethereum": "ethereum_smartcontract", - "polygon": "polygon_smartcontract", - "mumbai": "mumbai_smartcontract", - "xdai": "xdai_smartcontract", - "wyrm": "wyrm_smartcontract", - "zksync_era_testnet": "zksync_era_testnet_smartcontract", - "zksync_era": "zksync_era_smartcontract", - "zksync_era_sepolia": "zksync_era_sepolia_smartcontract", - "arbitrum_nova": "arbitrum_nova_smartcontract", - "arbitrum_sepolia": "arbitrum_sepolia_smartcontract", - "xai": "xai_smartcontract", - "xai_sepolia": "xai_sepolia_smartcontract", - "avalanche": "avalanche_smartcontract", - "avalanche_fuji": "avalanche_fuji_smartcontract", - "blast": "blast_smartcontract", - "blast_sepolia": "blast_sepolia_smartcontract", -} - -blockchain_by_subscription_id = { - "ethereum_blockchain": "ethereum", - "polygon_blockchain": "polygon", - "mumbai_blockchain": "mumbai", - "xdai_blockchain": "xdai", - "wyrm_blockchain": "wyrm", - "zksync_era_testnet_blockchain": "zksync_era_testnet", - "zksync_era_blockchain": "zksync_era", - "zksync_era_sepolia_blockchain": "zksync_era_sepolia", - "arbitrum_nova_blockchain": "arbitrum_nova", - "arbitrum_sepolia_blockchain": "arbitrum_sepolia", - "xai_blockchain": "xai", - "xai_sepolia_blockchain": "xai_sepolia", - "avalanche_blockchain": "avalanche", - "avalanche_fuji_blockchain": "avalanche_fuji", - "blast_blockchain": "blast", - "blast_sepolia_blockchain": "blast_sepolia", - "ethereum_smartcontract": "ethereum", - "polygon_smartcontract": "polygon", - "mumbai_smartcontract": "mumbai", - "xdai_smartcontract": "xdai", - "wyrm_smartcontract": "wyrm", - "zksync_era_testnet_smartcontract": "zksync_era_testnet", - "zksync_era_smartcontract": "zksync_era", - "zksync_era_sepolia_smartcontract": "zksync_era_sepolia", - "arbitrum_nova_smartcontract": "arbitrum_nova", - "arbitrum_sepolia_smartcontract": "arbitrum_sepolia", - "xai_smartcontract": "xai", - "xai_sepolia_smartcontract": "xai_sepolia", - "avalanche_smartcontract": "avalanche", - "avalanche_fuji_smartcontract": "avalanche_fuji", - "blast_smartcontract": "blast", - "blast_sepolia_smartcontract": "blast_sepolia", -} - - class TimeScale(Enum): # TODO(Andrey) Unlock when we be sure about perfomanse of agregation on transactions table. # Right now it can be hungs diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 11ddf645..6f915441 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.4.2" +MOONCRAWL_VERSION = "0.4.3" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a6972921..cce5e77f 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -24,6 +24,7 @@ export NFT_HUMBUG_TOKEN="" export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 0e9e5b8b..b271e111 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.12", + "moonstreamdb>=0.4.1", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug",