mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Amoy for mooncrawl

pull/1053/head
kompotkot 2024-04-15 08:05:26 +00:00
rodzic 3d81cb79fc
commit 525abb8a5f
11 zmienionych plików z 34 dodań i 223 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

3
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -22,6 +22,7 @@ from .data import DateRange
from .db import yield_db_session, yield_db_session_ctx
from .settings import (
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI,
MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI,
@ -66,6 +67,8 @@ def connect(
web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.MUMBAI:
web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.AMOY:
web3_uri = MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.XDAI:
web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:

3
crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py
Wyświetl plik

@ -4,6 +4,7 @@ from typing import Optional
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.subscriptions import blockchain_type_to_subscription_type
from web3 import Web3
from web3.middleware import geth_poa_middleware
@ -15,8 +16,6 @@ from ..settings import (
)
from .continuous_crawler import _retry_connect_web3, continuous_crawler
from .crawler import (
SubscriptionTypes,
blockchain_type_to_subscription_type,
find_all_deployed_blocks,
get_crawl_job_entries,
make_event_crawl_jobs,

42
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network
from moonstreamdb.networks import blockchain_type_to_network_type
from moonstreamdb.subscriptions import blockchain_type_to_subscription_type
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
@ -21,7 +22,6 @@ from .crawler import (
EventCrawlJob,
FunctionCallCrawlJob,
_retry_connect_web3,
blockchain_type_to_subscription_type,
get_crawl_job_entries,
heartbeat,
make_event_crawl_jobs,
@ -124,40 +124,10 @@ def continuous_crawler(
if web3 is None:
web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
if blockchain_type == AvailableBlockchainType.ETHEREUM:
network = Network.ethereum
elif blockchain_type == AvailableBlockchainType.POLYGON:
network = Network.polygon
elif blockchain_type == AvailableBlockchainType.MUMBAI:
network = Network.mumbai
elif blockchain_type == AvailableBlockchainType.XDAI:
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
network = Network.zksync_era_sepolia
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
elif blockchain_type == AvailableBlockchainType.XAI:
network = Network.xai
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
network = Network.xai_sepolia
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
try:
network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
except Exception as e:
raise Exception(e)
ethereum_state_provider = MoonstreamEthereumStateProvider(
web3,

59
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -11,6 +11,7 @@ from uuid import UUID
from bugout.data import BugoutJournalEntries, BugoutSearchResult
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.subscriptions import SubscriptionTypes
from moonworm.deployment import find_deployment_block # type: ignore
from web3.main import Web3
@ -29,25 +30,6 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class SubscriptionTypes(Enum):
POLYGON_BLOCKCHAIN = "polygon_smartcontract"
ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract"
MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
XDAI_BLOCKCHAIN = "xdai_smartcontract"
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN = "zksync_era_sepolia_smartcontract"
ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract"
XAI_BLOCKCHAIN = "xai_smartcontract"
XAI_SEPOLIA_BLOCKCHAIN = "xai_sepolia_smartcontract"
AVALANCHE_BLOCKCHAIN = "avalanche_smartcontract"
AVALANCHE_FUJI_BLOCKCHAIN = "avalanche_fuji_smartcontract"
BLAST_BLOCKCHAIN = "blast_smartcontract"
BLAST_SEPOLIA_BLOCKCHAIN = "blast_sepolia_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
"""
Stringifies a function ABI input object according to the ABI specification:
@ -137,45 +119,6 @@ def _retry_connect_web3(
)
def blockchain_type_to_subscription_type(
blockchain_type: AvailableBlockchainType,
) -> SubscriptionTypes:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
return SubscriptionTypes.ETHEREUM_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.POLYGON:
return SubscriptionTypes.POLYGON_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.MUMBAI:
return SubscriptionTypes.MUMBAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.XDAI:
return SubscriptionTypes.XDAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.WYRM:
return SubscriptionTypes.WYRM_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
return SubscriptionTypes.ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.XAI:
return SubscriptionTypes.XAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
return SubscriptionTypes.XAI_SEPOLIA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
return SubscriptionTypes.AVALANCHE_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
return SubscriptionTypes.AVALANCHE_FUJI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.BLAST:
return SubscriptionTypes.BLAST_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
return SubscriptionTypes.BLAST_SEPOLIA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
@dataclass
class EventCrawlJob:
event_abi_hash: str

40
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -2,7 +2,7 @@ import logging
from typing import List
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonstreamdb.networks import blockchain_type_to_network_type # type: ignore
from moonworm.crawler.function_call_crawler import ( # type: ignore
ContractFunctionCall,
FunctionCallCrawler,
@ -58,40 +58,10 @@ def function_call_crawler(
end_block: int,
batch_size: int,
):
if blockchain_type == AvailableBlockchainType.ETHEREUM:
network = Network.ethereum
elif blockchain_type == AvailableBlockchainType.POLYGON:
network = Network.polygon
elif blockchain_type == AvailableBlockchainType.MUMBAI:
network = Network.mumbai
elif blockchain_type == AvailableBlockchainType.XDAI:
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
network = Network.zksync_era_sepolia
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
elif blockchain_type == AvailableBlockchainType.XAI:
network = Network.xai
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
network = Network.xai_sepolia
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
try:
network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
except Exception as e:
raise Exception(e)
ethereum_state_provider = MoonstreamEthereumStateProvider(
web3,

40
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -5,7 +5,7 @@ from uuid import UUID
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonstreamdb.networks import blockchain_type_to_network_type # type: ignore
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
@ -52,40 +52,10 @@ def historical_crawler(
web3.eth.block_number >= start_block
), "start_block must be less than current block"
if blockchain_type == AvailableBlockchainType.ETHEREUM:
network = Network.ethereum
elif blockchain_type == AvailableBlockchainType.POLYGON:
network = Network.polygon
elif blockchain_type == AvailableBlockchainType.MUMBAI:
network = Network.mumbai
elif blockchain_type == AvailableBlockchainType.XDAI:
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
network = Network.zksync_era_sepolia
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
elif blockchain_type == AvailableBlockchainType.XAI:
network = Network.xai
elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
network = Network.xai_sepolia
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
network = Network.avalanche
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
network = Network.avalanche_fuji
elif blockchain_type == AvailableBlockchainType.BLAST:
network = Network.blast
elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
network = Network.blast_sepolia
else:
raise Exception("Unsupported blockchain type provided")
try:
network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
except Exception as e:
raise Exception(e)
ethereum_state_provider = MoonstreamEthereumStateProvider(
web3,

6
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -96,6 +96,12 @@ MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get(
if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", ""
)

59
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -24,6 +24,10 @@ from moonstreamdb.blockchain import (
get_label_model,
get_transaction_model,
)
from moonstreamdb.subscriptions import (
blockchain_by_subscription_id,
subscription_id_by_blockchain,
)
from sqlalchemy import and_
from sqlalchemy import cast as sqlalchemy_cast
from sqlalchemy import distinct, extract, func, text
@ -47,61 +51,6 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
subscription_id_by_blockchain = {
"ethereum": "ethereum_smartcontract",
"polygon": "polygon_smartcontract",
"mumbai": "mumbai_smartcontract",
"xdai": "xdai_smartcontract",
"wyrm": "wyrm_smartcontract",
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
"zksync_era": "zksync_era_smartcontract",
"zksync_era_sepolia": "zksync_era_sepolia_smartcontract",
"arbitrum_nova": "arbitrum_nova_smartcontract",
"arbitrum_sepolia": "arbitrum_sepolia_smartcontract",
"xai": "xai_smartcontract",
"xai_sepolia": "xai_sepolia_smartcontract",
"avalanche": "avalanche_smartcontract",
"avalanche_fuji": "avalanche_fuji_smartcontract",
"blast": "blast_smartcontract",
"blast_sepolia": "blast_sepolia_smartcontract",
}
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
"mumbai_blockchain": "mumbai",
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"zksync_era_blockchain": "zksync_era",
"zksync_era_sepolia_blockchain": "zksync_era_sepolia",
"arbitrum_nova_blockchain": "arbitrum_nova",
"arbitrum_sepolia_blockchain": "arbitrum_sepolia",
"xai_blockchain": "xai",
"xai_sepolia_blockchain": "xai_sepolia",
"avalanche_blockchain": "avalanche",
"avalanche_fuji_blockchain": "avalanche_fuji",
"blast_blockchain": "blast",
"blast_sepolia_blockchain": "blast_sepolia",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
"xdai_smartcontract": "xdai",
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
"zksync_era_smartcontract": "zksync_era",
"zksync_era_sepolia_smartcontract": "zksync_era_sepolia",
"arbitrum_nova_smartcontract": "arbitrum_nova",
"arbitrum_sepolia_smartcontract": "arbitrum_sepolia",
"xai_smartcontract": "xai",
"xai_sepolia_smartcontract": "xai_sepolia",
"avalanche_smartcontract": "avalanche",
"avalanche_fuji_smartcontract": "avalanche_fuji",
"blast_smartcontract": "blast",
"blast_sepolia_smartcontract": "blast_sepolia",
}
class TimeScale(Enum):
# TODO(Andrey) Unlock when we be sure about perfomanse of agregation on transactions table.
# Right now it can be hungs

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.4.2"
MOONCRAWL_VERSION = "0.4.3"

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -24,6 +24,7 @@ export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.12",
"moonstreamdb>=0.4.1",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",