mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1049 from moonstream-to/blast-crawlers-init 

Blast crawlers init
pull/1050/head
Sergei Sumarokov 2024-04-10 14:54:46 +03:00 zatwierdzone przez GitHub
rodzic 7994da519b f84788a6c1
commit 163049b9b6
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B5690EEEBB952194
21 zmienionych plików z 311 dodań i 11 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
crawlers/deploy/blast-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Blast database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain blast -n
CPUWeight=50
SyslogIdentifier=blast-missing

9
crawlers/deploy/blast-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Blast database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/blast-moonworm-crawler.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Blast moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b blast --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=blast-moonworm-crawler
[Install]
WantedBy=multi-user.target

11
crawlers/deploy/blast-sepolia-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Blast Sepolia database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain blast_sepolia -n
CPUWeight=50
SyslogIdentifier=blast-sepolia-missing

9
crawlers/deploy/blast-sepolia-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Blast Sepolia database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/blast-sepolia-moonworm-crawler.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Blast Sepolia moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b blast_sepolia --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=blast-sepolia-moonworm-crawler
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/blast-sepolia-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Blast Sepolia block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain blast_sepolia -c 10 -j 2
CPUWeight=90
SyslogIdentifier=blast-sepolia-synchronize
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/blast-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Blast block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain blast -c 10 -j 2
CPUWeight=90
SyslogIdentifier=blast-synchronize
[Install]
WantedBy=multi-user.target

46
crawlers/deploy/deploy-basic.bash
Wyświetl plik

@ -93,6 +93,16 @@ AVALANCHE_FUJI_SYNCHRONIZE_SERVICE="avalanche-fuji-synchronize.service"
AVALANCHE_FUJI_MISSING_SERVICE_FILE="avalanche-fuji-missing.service"
AVALANCHE_FUJI_MISSING_TIMER_FILE="avalanche-fuji-missing.timer"
# Blast
BLAST_MISSING_SERVICE_FILE="blast-missing.service"
BLAST_MISSING_TIMER_FILE="blast-missing.timer"
BLAST_SYNCHRONIZE_SERVICE="blast-synchronize.service"
# Blast sepolia
BLAST_SEPOLIA_MISSING_SERVICE_FILE="blast-sepolia-missing.service"
BLAST_SEPOLIA_MISSING_TIMER_FILE="blast-sepolia-missing.timer"
BLAST_SEPOLIA_SYNCHRONIZE_SERVICE="blast-sepolia-synchronize.service"
set -eu
echo
@ -370,3 +380,39 @@ cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/
cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MISSING_TIMER_FILE}"
# Blast
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast block with transactions syncronizer service definition with ${BLAST_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast missing service and timer with: ${BLAST_MISSING_SERVICE_FILE}, ${BLAST_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MISSING_TIMER_FILE}"
# Blast sepolia
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia block with transactions syncronizer service definition with ${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia missing service and timer with: ${BLAST_SEPOLIA_MISSING_SERVICE_FILE}, ${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MISSING_TIMER_FILE}"

18
crawlers/deploy/deploy-moonworm.bash
Wyświetl plik

@ -37,6 +37,8 @@ XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service"
XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="xai-sepolia-moonworm-crawler.service"
AVALANCHE_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-moonworm-crawler.service"
AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-fuji-moonworm-crawler.service"
BLAST_MOONWORM_CRAWLER_SERVICE_FILE="blast-moonworm-crawler.service"
BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="blast-sepolia-moonworm-crawler.service"
set -eu
@ -172,3 +174,19 @@ chmod 644 "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast moonworm crawler service definition with ${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia moonworm crawler service definition with ${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"

70
crawlers/deploy/deploy.bash
Wyświetl plik

@ -139,18 +139,30 @@ XAI_SEPOLIA_MISSING_TIMER_FILE="xai-sepolia-missing.timer"
XAI_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="xai-sepolia-moonworm-crawler.service"
XAI_SEPOLIA_SYNCHRONIZE_SERVICE="xai-sepolia-synchronize.service"
# Avalanche sepolia
# Avalanche
AVALANCHE_SYNCHRONIZE_SERVICE="avalanche-synchronize.service"
AVALANCHE_MISSING_SERVICE_FILE="avalanche-missing.service"
AVALANCHE_MISSING_TIMER_FILE="avalanche-missing.timer"
AVALANCHE_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-moonworm-crawler.service"
# Avalanche Fuji sepolia
# Avalanche Fuji
AVALANCHE_FUJI_SYNCHRONIZE_SERVICE="avalanche-fuji-synchronize.service"
AVALANCHE_FUJI_MISSING_SERVICE_FILE="avalanche-fuji-missing.service"
AVALANCHE_FUJI_MISSING_TIMER_FILE="avalanche-fuji-missing.timer"
AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE="avalanche-fuji-moonworm-crawler.service"
# Blast
BLAST_MISSING_SERVICE_FILE="blast-missing.service"
BLAST_MISSING_TIMER_FILE="blast-missing.timer"
BLAST_MOONWORM_CRAWLER_SERVICE_FILE="blast-moonworm-crawler.service"
BLAST_SYNCHRONIZE_SERVICE="blast-synchronize.service"
# Blast sepolia
BLAST_SEPOLIA_MISSING_SERVICE_FILE="blast-sepolia-missing.service"
BLAST_SEPOLIA_MISSING_TIMER_FILE="blast-sepolia-missing.timer"
BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="blast-sepolia-moonworm-crawler.service"
BLAST_SEPOLIA_SYNCHRONIZE_SERVICE="blast-sepolia-synchronize.service"
set -eu
echo
@ -641,8 +653,6 @@ cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubun
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
# Xai
echo
echo
@ -747,3 +757,55 @@ chmod 644 "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${AVALANCHE_FUJI_MOONWORM_CRAWLER_SERVICE_FILE}"
# Blast
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast block with transactions syncronizer service definition with ${BLAST_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${BLAST_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast missing service and timer with: ${BLAST_MISSING_SERVICE_FILE}, ${BLAST_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast moonworm crawler service definition with ${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_MOONWORM_CRAWLER_SERVICE_FILE}"
# Blast sepolia
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia block with transactions syncronizer service definition with ${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia missing service and timer with: ${BLAST_SEPOLIA_MISSING_SERVICE_FILE}, ${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Blast sepolia moonworm crawler service definition with ${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${BLAST_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"

23
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -26,6 +26,8 @@ from .settings import (
MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI,
MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI,
MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI,
MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI,
MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI,
MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI,
@ -82,6 +84,10 @@ def connect(
web3_uri = MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
web3_uri = MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.BLAST:
web3_uri = MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
web3_uri = MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -146,7 +152,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
)
if blockchain_type == AvailableBlockchainType.XDAI:
block_obj.author = block.author
if (
elif (
blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
@ -163,7 +169,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
if block.get("l1BatchTimestamp") is not None
else None
)
if (
elif (
blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA
or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA
):
@ -172,8 +178,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
if (
elif (
blockchain_type == AvailableBlockchainType.XAI
or blockchain_type == AvailableBlockchainType.XAI_SEPOLIA
):
@ -182,8 +187,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
if (
elif (
blockchain_type == AvailableBlockchainType.AVALANCHE
or blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI
):
@ -192,6 +196,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.block_gas_cost = block.get("blockGasCost")
block_obj.ext_data_gas_used = block.get("extDataGasUsed", "")
block_obj.ext_data_hash = block.get("extDataHash", "")
elif (
blockchain_type == AvailableBlockchainType.BLAST
or blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA
):
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.mix_hash = block.get("mixHash", "")
block_obj.withdrawals_root = block.get("withdrawalsRoot", "")
db_session.add(block_obj)

4
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -152,6 +152,10 @@ def continuous_crawler(
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -44,6 +44,8 @@ class SubscriptionTypes(Enum):
XAI_SEPOLIA_BLOCKCHAIN = "xai_sepolia_smartcontract"
AVALANCHE_BLOCKCHAIN = "avalanche_smartcontract"
AVALANCHE_FUJI_BLOCKCHAIN = "avalanche_fuji_smartcontract"
BLAST_BLOCKCHAIN = "blast_smartcontract"
BLAST_SEPOLIA_BLOCKCHAIN = "blast_sepolia_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -166,6 +168,10 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.AVALANCHE_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
return SubscriptionTypes.AVALANCHE_FUJI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.BLAST:
return SubscriptionTypes.BLAST_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
return SubscriptionTypes.BLAST_SEPOLIA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

10
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -72,6 +72,8 @@ def function_call_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
network = Network.zksync_era_sepolia
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
@ -80,6 +82,14 @@ def function_call_crawler(
network = Network.xai
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
network = Network.xai_sepolia
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

10
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -66,6 +66,8 @@ def historical_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
network = Network.zksync_era_sepolia
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
@ -74,6 +76,14 @@ def historical_crawler(
network = Network.xai
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
network = Network.xai_sepolia
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise Exception("Unsupported blockchain type provided")

14
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -160,6 +160,20 @@ if MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI == "":
"MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

6
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -62,6 +62,8 @@ subscription_id_by_blockchain = {
"xai_sepolia": "xai_sepolia_smartcontract",
"avalanche": "avalanche_smartcontract",
"avalanche_fuji": "avalanche_fuji_smartcontract",
"blast": "blast_smartcontract",
"blast_sepolia": "blast_sepolia_smartcontract",
}
blockchain_by_subscription_id = {
@ -79,6 +81,8 @@ blockchain_by_subscription_id = {
"xai_sepolia_blockchain": "xai_sepolia",
"avalanche_blockchain": "avalanche",
"avalanche_fuji_blockchain": "avalanche_fuji",
"blast_blockchain": "blast",
"blast_sepolia_blockchain": "blast_sepolia",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
@ -93,6 +97,8 @@ blockchain_by_subscription_id = {
"xai_sepolia_smartcontract": "xai_sepolia",
"avalanche_smartcontract": "avalanche",
"avalanche_fuji_smartcontract": "avalanche_fuji",
"blast_smartcontract": "blast",
"blast_sepolia_smartcontract": "blast_sepolia",
}

3
crawlers/mooncrawl/mypy.ini
Wyświetl plik

@ -11,3 +11,6 @@ ignore_missing_imports = True
[mypy-tqdm.*]
ignore_missing_imports = True
[mypy-moonworm.*]
ignore_missing_imports = True

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -33,6 +33,8 @@ export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>
export MOONSTREAM_NODE_XAI_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_BLAST_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_BLAST_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
# AWS environment variables
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.11",
"moonstreamdb>=0.3.12",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",