diff --git a/crawlers/deploy/blast-missing.service b/crawlers/deploy/blast-missing.service new file mode 100644 index 00000000..6d08f899 --- /dev/null +++ b/crawlers/deploy/blast-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Blast database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain blast -n +CPUWeight=50 +SyslogIdentifier=blast-missing \ No newline at end of file diff --git a/crawlers/deploy/blast-missing.timer b/crawlers/deploy/blast-missing.timer new file mode 100644 index 00000000..1a3c93a4 --- /dev/null +++ b/crawlers/deploy/blast-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Blast database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/blast-moonworm-crawler.service b/crawlers/deploy/blast-moonworm-crawler.service new file mode 100644 index 00000000..42729bf0 --- /dev/null +++ b/crawlers/deploy/blast-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Blast moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b blast --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=blast-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/blast-sepolia-missing.service b/crawlers/deploy/blast-sepolia-missing.service new file mode 100644 index 00000000..ed67a323 --- /dev/null +++ b/crawlers/deploy/blast-sepolia-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Blast Sepolia database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain blast_sepolia -n +CPUWeight=50 +SyslogIdentifier=blast-sepolia-missing \ No newline at end of file diff --git a/crawlers/deploy/blast-sepolia-missing.timer b/crawlers/deploy/blast-sepolia-missing.timer new file mode 100644 index 00000000..2e05b82b --- /dev/null +++ b/crawlers/deploy/blast-sepolia-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Blast Sepolia database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/blast-sepolia-moonworm-crawler.service b/crawlers/deploy/blast-sepolia-moonworm-crawler.service new file mode 100644 index 00000000..d6898db8 --- /dev/null +++ b/crawlers/deploy/blast-sepolia-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Blast Sepolia moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b blast_sepolia --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=blast-sepolia-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/blast-sepolia-synchronize.service b/crawlers/deploy/blast-sepolia-synchronize.service new file mode 100644 index 00000000..f24a8860 --- /dev/null +++ b/crawlers/deploy/blast-sepolia-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Blast Sepolia block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain blast_sepolia -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=blast-sepolia-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/blast-synchronize.service b/crawlers/deploy/blast-synchronize.service new file mode 100644 index 00000000..bc63f2bf --- /dev/null +++ b/crawlers/deploy/blast-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Blast block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain blast -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=blast-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/deploy-basic.bash b/crawlers/deploy/deploy-basic.bash index 1e688961..04598125 100755 --- a/crawlers/deploy/deploy-basic.bash +++ b/crawlers/deploy/deploy-basic.bash @@ -93,6 +93,16 @@ AVALANCHE_FUJI_SYNCHRONIZE_SERVICE="avalanche-fuji-synchronize.service" AVALANCHE_FUJI_MISSING_SERVICE_FILE="avalanche-fuji-missing.service" AVALANCHE_FUJI_MISSING_TIMER_FILE="avalanche-fuji-missing.timer" +# Blast +BLAST_MISSING_SERVICE_FILE="blast-missing.service" +BLAST_MISSING_TIMER_FILE="blast-missing.timer" +BLAST_SYNCHRONIZE_SERVICE="blast-synchronize.service" + +# Blast sepolia +BLAST_SEPOLIA_MISSING_SERVICE_FILE="blast-sepolia-missing.service" +BLAST_SEPOLIA_MISSING_TIMER_FILE="blast-sepolia-missing.timer" +BLAST_SEPOLIA_SYNCHRONIZE_SERVICE="blast-sepolia-synchronize.service" + set -eu echo @@ -370,3 +380,39 @@ cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/ cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MISSING_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MISSING_TIMER_FILE}" + +# Blast +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast block with transactions syncronizer service definition with ${BLAST_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast missing service and timer with: ${BLAST_MISSING_SERVICE_FILE}, ${BLAST_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MISSING_TIMER_FILE}" + +# Blast sepolia +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia block with transactions syncronizer service definition with ${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia missing service and timer with: ${BLAST_SEPOLIA_MISSING_SERVICE_FILE}, ${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MISSING_TIMER_FILE}" diff --git a/crawlers/deploy/deploy-moonworm.bash b/crawlers/deploy/deploy-moonworm.bash index 6fa2f00e..f2bf1ceb 100755 --- a/crawlers/deploy/deploy-moonworm.bash +++ b/crawlers/deploy/deploy-moonworm.bash @@ -37,6 +37,8 @@ XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service" XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="xai-sepolia-moonworm-crawler.service" AVALANCHE_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-moonworm-crawler.service" AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-fuji-moonworm-crawler.service" +BLAST_MOONWORM_CRAWLER_SERVICE_FILE="blast-moonworm-crawler.service" +BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="blast-sepolia-moonworm-crawler.service" set -eu @@ -172,3 +174,19 @@ chmod 644 "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast moonworm crawler service definition with ${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia moonworm crawler service definition with ${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index dad70072..af32e1c5 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -139,18 +139,30 @@ XAI_SEPOLIA_MISSING_TIMER_FILE="xai-sepolia-missing.timer" XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="xai-sepolia-moonworm-crawler.service" XAI_SEPOLIA_SYNCHRONIZE_SERVICE="xai-sepolia-synchronize.service" -# Avalanche sepolia +# Avalanche AVALANCHE_SYNCHRONIZE_SERVICE="avalanche-synchronize.service" AVALANCHE_MISSING_SERVICE_FILE="avalanche-missing.service" AVALANCHE_MISSING_TIMER_FILE="avalanche-missing.timer" AVALANCHE_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-moonworm-crawler.service" -# Avalanche Fuji sepolia +# Avalanche Fuji AVALANCHE_FUJI_SYNCHRONIZE_SERVICE="avalanche-fuji-synchronize.service" AVALANCHE_FUJI_MISSING_SERVICE_FILE="avalanche-fuji-missing.service" AVALANCHE_FUJI_MISSING_TIMER_FILE="avalanche-fuji-missing.timer" AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-fuji-moonworm-crawler.service" +# Blast +BLAST_MISSING_SERVICE_FILE="blast-missing.service" +BLAST_MISSING_TIMER_FILE="blast-missing.timer" +BLAST_MOONWORM_CRAWLER_SERVICE_FILE="blast-moonworm-crawler.service" +BLAST_SYNCHRONIZE_SERVICE="blast-synchronize.service" + +# Blast sepolia +BLAST_SEPOLIA_MISSING_SERVICE_FILE="blast-sepolia-missing.service" +BLAST_SEPOLIA_MISSING_TIMER_FILE="blast-sepolia-missing.timer" +BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="blast-sepolia-moonworm-crawler.service" +BLAST_SEPOLIA_SYNCHRONIZE_SERVICE="blast-sepolia-synchronize.service" + set -eu echo @@ -641,8 +653,6 @@ cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubun XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" - - # Xai echo echo @@ -747,3 +757,55 @@ chmod 644 "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" + +# Blast +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast block with transactions syncronizer service definition with ${BLAST_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast missing service and timer with: ${BLAST_MISSING_SERVICE_FILE}, ${BLAST_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast moonworm crawler service definition with ${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" + +# Blast sepolia +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia block with transactions syncronizer service definition with ${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia missing service and timer with: ${BLAST_SEPOLIA_MISSING_SERVICE_FILE}, ${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Blast sepolia moonworm crawler service definition with ${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index e5e97564..2f2bd95a 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -26,6 +26,8 @@ from .settings import ( MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI, MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI, MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI, + MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI, + MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI, MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI, MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI, MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI, @@ -82,6 +84,10 @@ def connect( web3_uri = MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: web3_uri = MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.BLAST: + web3_uri = MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: + web3_uri = MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -146,7 +152,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) ) if blockchain_type == AvailableBlockchainType.XDAI: block_obj.author = block.author - if ( + elif ( blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA @@ -163,7 +169,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) if block.get("l1BatchTimestamp") is not None else None ) - if ( + elif ( blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA ): @@ -172,8 +178,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_count = hex_to_int(block.get("sendCount")) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") - - if ( + elif ( blockchain_type == AvailableBlockchainType.XAI or blockchain_type == AvailableBlockchainType.XAI_SEPOLIA ): @@ -182,8 +187,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_count = hex_to_int(block.get("sendCount")) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") - - if ( + elif ( blockchain_type == AvailableBlockchainType.AVALANCHE or blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI ): @@ -192,6 +196,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.block_gas_cost = block.get("blockGasCost") block_obj.ext_data_gas_used = block.get("extDataGasUsed", "") block_obj.ext_data_hash = block.get("extDataHash", "") + elif ( + blockchain_type == AvailableBlockchainType.BLAST + or blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA + ): + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.mix_hash = block.get("mixHash", "") + block_obj.withdrawals_root = block.get("withdrawalsRoot", "") db_session.add(block_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 823e6dda..990200a8 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -152,6 +152,10 @@ def continuous_crawler( network = Network.avalanche elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: network = Network.avalanche_fuji + elif blockchain_type == AvailableBlockchainType.BLAST: + network = Network.blast + elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: + network = Network.blast_sepolia else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 6bb1efd8..e90910f1 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -44,6 +44,8 @@ class SubscriptionTypes(Enum): XAI_SEPOLIA_BLOCKCHAIN = "xai_sepolia_smartcontract" AVALANCHE_BLOCKCHAIN = "avalanche_smartcontract" AVALANCHE_FUJI_BLOCKCHAIN = "avalanche_fuji_smartcontract" + BLAST_BLOCKCHAIN = "blast_smartcontract" + BLAST_SEPOLIA_BLOCKCHAIN = "blast_sepolia_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -166,6 +168,10 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.AVALANCHE_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: return SubscriptionTypes.AVALANCHE_FUJI_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.BLAST: + return SubscriptionTypes.BLAST_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: + return SubscriptionTypes.BLAST_SEPOLIA_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 3ef05095..44834fdf 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -72,6 +72,8 @@ def function_call_crawler( network = Network.zksync_era_testnet elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: network = Network.zksync_era + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + network = Network.zksync_era_sepolia elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: network = Network.arbitrum_nova elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: @@ -80,6 +82,14 @@ def function_call_crawler( network = Network.xai elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: network = Network.xai_sepolia + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + network = Network.avalanche + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + network = Network.avalanche_fuji + elif blockchain_type == AvailableBlockchainType.BLAST: + network = Network.blast + elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: + network = Network.blast_sepolia else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 1fa3aa90..6761616f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -66,6 +66,8 @@ def historical_crawler( network = Network.zksync_era_testnet elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: network = Network.zksync_era + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + network = Network.zksync_era_sepolia elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: network = Network.arbitrum_nova elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: @@ -74,6 +76,14 @@ def historical_crawler( network = Network.xai elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA: network = Network.xai_sepolia + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + network = Network.avalanche + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + network = Network.avalanche_fuji + elif blockchain_type == AvailableBlockchainType.BLAST: + network = Network.blast + elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA: + network = Network.blast_sepolia else: raise Exception("Unsupported blockchain type provided") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 9c479fcf..99d604ff 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -160,6 +160,20 @@ if MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI == "": "MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI env variable is not set" ) +MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI env variable is not set") + +MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI == "": + raise Exception( + "MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI env variable is not set" + ) + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 459f7653..74ca3375 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -62,6 +62,8 @@ subscription_id_by_blockchain = { "xai_sepolia": "xai_sepolia_smartcontract", "avalanche": "avalanche_smartcontract", "avalanche_fuji": "avalanche_fuji_smartcontract", + "blast": "blast_smartcontract", + "blast_sepolia": "blast_sepolia_smartcontract", } blockchain_by_subscription_id = { @@ -79,6 +81,8 @@ blockchain_by_subscription_id = { "xai_sepolia_blockchain": "xai_sepolia", "avalanche_blockchain": "avalanche", "avalanche_fuji_blockchain": "avalanche_fuji", + "blast_blockchain": "blast", + "blast_sepolia_blockchain": "blast_sepolia", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", @@ -93,6 +97,8 @@ blockchain_by_subscription_id = { "xai_sepolia_smartcontract": "xai_sepolia", "avalanche_smartcontract": "avalanche", "avalanche_fuji_smartcontract": "avalanche_fuji", + "blast_smartcontract": "blast", + "blast_sepolia_smartcontract": "blast_sepolia", } diff --git a/crawlers/mooncrawl/mypy.ini b/crawlers/mooncrawl/mypy.ini index 45381262..0a92316e 100644 --- a/crawlers/mooncrawl/mypy.ini +++ b/crawlers/mooncrawl/mypy.ini @@ -11,3 +11,6 @@ ignore_missing_imports = True [mypy-tqdm.*] ignore_missing_imports = True + +[mypy-moonworm.*] +ignore_missing_imports = True diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 9819b89f..a6972921 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -33,6 +33,8 @@ export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https:// export MOONSTREAM_NODE_XAI_SEPOLIA_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI="https://" # AWS environment variables export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index ce99baa9..0e9e5b8b 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.11", + "moonstreamdb>=0.3.12", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug",