Merge branch 'main' into metadata-crawler

pull/660/head
Andrey 2022-09-08 17:42:40 +03:00
commit 14dedab479
13 zmienionych plików z 1225 dodań i 40 usunięć

Wyświetl plik

@ -45,6 +45,8 @@ POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service"
POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
POLYGON_STATE_SERVICE_FILE="polygon-state.service"
POLYGON_STATE_TIMER_FILE="polygon-state.timer"
# XDai service file
XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service"
@ -212,3 +214,12 @@ chmod 644 "${SCRIPT_DIR}/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}" "/etc/systemd/system/${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart --no-block "${XDAI_MOONWORM_CRAWLER_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon state service and timer with: ${POLYGON_STATE_SERVICE_FILE}, ${POLYGON_STATE_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${POLYGON_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_STATE_TIMER_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_STATE_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_STATE_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_STATE_TIMER_FILE}" "/etc/systemd/system/${POLYGON_STATE_TIMER_FILE}"
systemctl daemon-reload
systemctl restart --no-block "${POLYGON_STATE_TIMER_FILE}"

Wyświetl plik

@ -0,0 +1,13 @@
[Unit]
Description=Execute state crawler
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --blockchain polygon
CPUWeight=60
SyslogIdentifier=polygon-state

Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Execute Polygon state crawler each 10m
[Timer]
OnBootSec=10s
OnUnitActiveSec=10m
[Install]
WantedBy=timers.target

Wyświetl plik

@ -1,4 +1,5 @@
import logging
import json
from typing import Dict, List, Optional
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
@ -20,13 +21,19 @@ def _event_to_label(
Creates a label model.
"""
label_model = get_label_model(blockchain_type)
sanityzed_label_data = json.loads(
json.dumps(
{
"type": "event",
"name": event.event_name,
"args": event.args,
}
).replace(r"\u0000", "")
)
label = label_model(
label=label_name,
label_data={
"type": "event",
"name": event.event_name,
"args": event.args,
},
label_data=sanityzed_label_data,
address=event.address,
block_number=event.block_number,
block_timestamp=event.block_timestamp,
@ -45,16 +52,23 @@ def _function_call_to_label(
Creates a label model.
"""
label_model = get_label_model(blockchain_type)
sanityzed_label_data = json.loads(
json.dumps(
{
"type": "tx_call",
"name": function_call.function_name,
"caller": function_call.caller_address,
"args": function_call.function_args,
"status": function_call.status,
"gasUsed": function_call.gas_used,
}
).replace(r"\u0000", "")
)
label = label_model(
label=label_name,
label_data={
"type": "tx_call",
"name": function_call.function_name,
"caller": function_call.caller_address,
"args": function_call.function_args,
"status": function_call.status,
"gasUsed": function_call.gas_used,
},
label_data=sanityzed_label_data,
address=function_call.contract_address,
block_number=function_call.block_number,
transaction_hash=function_call.transaction_hash,

Wyświetl plik

@ -1,8 +1,10 @@
import os
from typing import Optional, cast
from typing import Optional, Dict
from uuid import UUID
from bugout.app import Bugout
from moonstreamdb.blockchain import AvailableBlockchainType
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION = "subscription"
BUGOUT_RESOURCE_TYPE_DASHBOARD = "dashboards"
@ -43,7 +45,6 @@ CRAWLER_LABEL = "moonworm-alpha"
VIEW_STATE_CRAWLER_LABEL = "view-state-alpha"
METADATA_CRAWLER_LABEL = "metadata-crawler"
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS = 30000
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS_RAW = os.environ.get(
"MOONSTREAM_QUERY_API_DB_STATEMENT_TIMEOUT_MILLIS"
@ -168,3 +169,10 @@ try:
NB_CONTROLLER_ACCESS_ID = UUID(NB_CONTROLLER_ACCESS_ID_RAW)
except:
pass
multicall_contracts: Dict[AvailableBlockchainType, str] = {
AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6",
AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631",
AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696",
}

Wyświetl plik

@ -0,0 +1,313 @@
[
{
"inputs": [
{
"components": [
{
"internalType": "address",
"name": "target",
"type": "address"
},
{
"internalType": "bytes",
"name": "callData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "aggregate",
"outputs": [
{
"internalType": "uint256",
"name": "blockNumber",
"type": "uint256"
},
{
"internalType": "bytes[]",
"name": "returnData",
"type": "bytes[]"
}
],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"components": [
{
"internalType": "address",
"name": "target",
"type": "address"
},
{
"internalType": "bytes",
"name": "callData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "blockAndAggregate",
"outputs": [
{
"internalType": "uint256",
"name": "blockNumber",
"type": "uint256"
},
{
"internalType": "bytes32",
"name": "blockHash",
"type": "bytes32"
},
{
"components": [
{
"internalType": "bool",
"name": "success",
"type": "bool"
},
{
"internalType": "bytes",
"name": "returnData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "uint256",
"name": "blockNumber",
"type": "uint256"
}
],
"name": "getBlockHash",
"outputs": [
{
"internalType": "bytes32",
"name": "blockHash",
"type": "bytes32"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getBlockNumber",
"outputs": [
{
"internalType": "uint256",
"name": "blockNumber",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockCoinbase",
"outputs": [
{
"internalType": "address",
"name": "coinbase",
"type": "address"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockDifficulty",
"outputs": [
{
"internalType": "uint256",
"name": "difficulty",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockGasLimit",
"outputs": [
{
"internalType": "uint256",
"name": "gaslimit",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockTimestamp",
"outputs": [
{
"internalType": "uint256",
"name": "timestamp",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "address",
"name": "addr",
"type": "address"
}
],
"name": "getEthBalance",
"outputs": [
{
"internalType": "uint256",
"name": "balance",
"type": "uint256"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getLastBlockHash",
"outputs": [
{
"internalType": "bytes32",
"name": "blockHash",
"type": "bytes32"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "bool",
"name": "requireSuccess",
"type": "bool"
},
{
"components": [
{
"internalType": "address",
"name": "target",
"type": "address"
},
{
"internalType": "bytes",
"name": "callData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "tryAggregate",
"outputs": [
{
"components": [
{
"internalType": "bool",
"name": "success",
"type": "bool"
},
{
"internalType": "bytes",
"name": "returnData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "nonpayable",
"type": "function"
},
{
"inputs": [
{
"internalType": "bool",
"name": "requireSuccess",
"type": "bool"
},
{
"components": [
{
"internalType": "address",
"name": "target",
"type": "address"
},
{
"internalType": "bytes",
"name": "callData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "tryBlockAndAggregate",
"outputs": [
{
"internalType": "uint256",
"name": "blockNumber",
"type": "uint256"
},
{
"internalType": "bytes32",
"name": "blockHash",
"type": "bytes32"
},
{
"components": [
{
"internalType": "bool",
"name": "success",
"type": "bool"
},
{
"internalType": "bytes",
"name": "returnData",
"type": "bytes"
}
],
"internalType": "struct Multicall2.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "nonpayable",
"type": "function"
}
]

Wyświetl plik

@ -0,0 +1,82 @@
# Code generated by moonworm : https://github.com/bugout-dev/moonworm
# Moonworm version : 0.2.4
import json
import os
from typing import Any, Dict, Union
from eth_typing.evm import Address, ChecksumAddress
from web3 import Web3
from web3.contract import ContractFunction
from .web3_util import *
abi_path = os.path.join(os.path.dirname(__file__), "Multicall2_abi.json")
with open(abi_path, "r") as abi_file:
CONTRACT_ABI = json.load(abi_file)
class Contract:
def __init__(self, web3: Web3, contract_address: ChecksumAddress):
self.web3 = web3
self.address = contract_address
self.contract = web3.eth.contract(address=self.address, abi=CONTRACT_ABI)
@staticmethod
def constructor() -> ContractConstructor:
return ContractConstructor()
def aggregate(self, calls: List) -> ContractFunction:
return self.contract.functions.aggregate(calls)
def blockAndAggregate(self, calls: List) -> ContractFunction:
return self.contract.functions.blockAndAggregate(calls)
def getBlockHash(self, blockNumber: int) -> ContractFunction:
return self.contract.functions.getBlockHash(blockNumber)
def getBlockNumber(self) -> ContractFunction:
return self.contract.functions.getBlockNumber()
def getCurrentBlockCoinbase(self) -> ContractFunction:
return self.contract.functions.getCurrentBlockCoinbase()
def getCurrentBlockDifficulty(self) -> ContractFunction:
return self.contract.functions.getCurrentBlockDifficulty()
def getCurrentBlockGasLimit(self) -> ContractFunction:
return self.contract.functions.getCurrentBlockGasLimit()
def getCurrentBlockTimestamp(self) -> ContractFunction:
return self.contract.functions.getCurrentBlockTimestamp()
def getEthBalance(self, addr: ChecksumAddress) -> ContractFunction:
return self.contract.functions.getEthBalance(addr)
def getLastBlockHash(self) -> ContractFunction:
return self.contract.functions.getLastBlockHash()
def tryAggregate(self, requireSuccess: bool, calls: List) -> ContractFunction:
return self.contract.functions.tryAggregate(requireSuccess, calls)
def tryBlockAndAggregate(
self, requireSuccess: bool, calls: List
) -> ContractFunction:
return self.contract.functions.tryBlockAndAggregate(requireSuccess, calls)
def deploy(
web3: Web3,
contract_constructor: ContractFunction,
contract_bytecode: str,
deployer_address: ChecksumAddress,
deployer_private_key: str,
) -> Contract:
tx_hash, contract_address = deploy_contract_from_constructor_function(
web3,
constructor=contract_constructor,
contract_bytecode=contract_bytecode,
contract_abi=CONTRACT_ABI,
deployer=deployer_address,
deployer_private_key=deployer_private_key,
)
return Contract(web3, contract_address)

Wyświetl plik

@ -0,0 +1,452 @@
import argparse
import json
import hashlib
import itertools
import logging
from typing import Dict, List, Any, Optional
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3
from moonstreamdb.db import (
MOONSTREAM_DB_URI,
MOONSTREAM_POOL_SIZE,
create_moonstream_engine,
)
from sqlalchemy.orm import sessionmaker
from .db import view_call_to_label, commit_session
from .Multicall2_interface import Contract as Multicall2
from ..settings import (
NB_CONTROLLER_ACCESS_ID,
MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
multicall_contracts,
)
from .web3_util import FunctionSignature
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
Multicall2_address = "0xc8E51042792d7405184DfCa245F2d27B94D013b6"
def make_multicall(
multicall_method: Any,
calls: List[Any],
block_timestamp: int,
block_number: str = "latest",
) -> Any:
multicall_calls = [
(
call["address"],
call["method"].encode_data(call["inputs"]).hex(),
)
for call in calls
]
multicall_result = multicall_method(False, calls=multicall_calls).call(
block_identifier=block_number
)
results = []
# Handle the case with not successful calls
for index, encoded_data in enumerate(multicall_result):
if encoded_data[0]:
results.append(
{
"result": calls[index]["method"].decode_data(encoded_data[1])[0],
"hash": calls[index]["hash"],
"method": calls[index]["method"],
"address": calls[index]["address"],
"name": calls[index]["method"].name,
"inputs": calls[index]["inputs"],
"call_data": multicall_calls[index][1],
"block_number": block_number,
"block_timestamp": block_timestamp,
"status": encoded_data[0],
}
)
else:
results.append(
{
"result": calls[index]["method"].decode_data(encoded_data[1]),
"hash": calls[index]["hash"],
"method": calls[index]["method"],
"address": calls[index]["address"],
"name": calls[index]["method"].name,
"inputs": calls[index]["inputs"],
"call_data": multicall_calls[index][1],
"block_number": block_number,
"block_timestamp": block_timestamp,
"status": encoded_data[0],
}
)
return results
def crawl_calls_level(
db_session,
calls,
responces,
contracts_ABIs,
interfaces,
batch_size,
multicall_method,
block_number,
blockchain_type,
block_timestamp,
):
calls_of_level = []
for call in calls:
parameters = []
for input in call["inputs"]:
if type(input["value"]) in (str, int):
if input["value"] not in responces:
parameters.append([input["value"]])
else:
if (
contracts_ABIs[call["address"]][input["value"]]["name"]
== "totalSupply"
):
parameters.append(
list(range(1, responces[input["value"]][0] + 1))
)
else:
parameters.append(responces[input["value"]])
elif type(input["value"]) == list:
parameters.append(input["value"])
else:
raise
for call_parameters in itertools.product(*parameters):
calls_of_level.append(
{
"address": call["address"],
"method": FunctionSignature(
interfaces[call["address"]].get_function_by_name(call["name"])
),
"hash": call["generated_hash"],
"inputs": call_parameters,
}
)
for call_chunk in [
calls_of_level[i : i + batch_size]
for i in range(0, len(calls_of_level), batch_size)
]:
while True:
try:
make_multicall_result = make_multicall(
multicall_method=multicall_method,
calls=call_chunk,
block_number=block_number,
block_timestamp=block_timestamp,
)
break
except ValueError:
continue
# results parsing and writing to database
add_to_session_count = 0
for result in make_multicall_result:
db_view = view_call_to_label(blockchain_type, result)
db_session.add(db_view)
add_to_session_count += 1
if result["hash"] not in responces:
responces[result["hash"]] = []
responces[result["hash"]].append(result["result"])
commit_session(db_session)
logger.info(f"{add_to_session_count} labels commit to database.")
def parse_jobs(
jobs: List[Any],
blockchain_type: AvailableBlockchainType,
block_number: Optional[int],
batch_size: int,
access_id: UUID,
):
"""
Parse jobs from list and generate web3 interfaces for each contract.
"""
contracts_ABIs: Dict[str, Any] = {}
contracts_methods: Dict[str, Any] = {}
calls: Dict[int, Any] = {0: []}
web3_client = _retry_connect_web3(
blockchain_type=blockchain_type, access_id=access_id
)
logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
if block_number is None:
block_number = web3_client.eth.get_block("latest").number # type: ignore
logger.info(f"Current block number: {block_number}")
block_timestamp = web3_client.eth.get_block(block_number).timestamp # type: ignore
multicaller = Multicall2(
web3_client, web3_client.toChecksumAddress(multicall_contracts[blockchain_type])
)
multicall_method = multicaller.tryAggregate
def recursive_unpack(method_abi: Any, level: int = 0) -> Any:
"""
Generate tree of calls for crawling
"""
have_subcalls = False
abi = {
"inputs": [],
"outputs": method_abi["outputs"],
"name": method_abi["name"],
"type": "function",
"stateMutability": "view",
}
for input in method_abi["inputs"]:
if type(input["value"]) in (str, int, list):
abi["inputs"].append(input)
elif type(input["value"]) == dict:
if input["value"]["type"] == "function":
hash_link = recursive_unpack(input["value"], level + 1)
# replace defenition by hash pointing to the result of the recursive_unpack
input["value"] = hash_link
have_subcalls = True
abi["inputs"].append(input)
abi["address"] = method_abi["address"]
generated_hash = hashlib.md5(json.dumps(abi).encode("utf-8")).hexdigest()
abi["generated_hash"] = generated_hash
if have_subcalls:
level += 1
if not calls.get(level):
calls[level] = []
calls[level].append(abi)
else:
level = 0
if not calls.get(level):
calls[level] = []
calls[level].append(abi)
if not contracts_methods.get(job["address"]):
contracts_methods[job["address"]] = []
if generated_hash not in contracts_methods[job["address"]]:
contracts_methods[job["address"]].append(generated_hash)
if not contracts_ABIs.get(job["address"]):
contracts_ABIs[job["address"]] = {}
contracts_ABIs[job["address"]][generated_hash] = abi
return generated_hash
for job in jobs:
if job["address"] not in contracts_ABIs:
contracts_ABIs[job["address"]] = []
recursive_unpack(job, 0)
# generate contracts interfaces
interfaces = {}
for contract_address in contracts_ABIs:
# collect abis for each contract
abis = []
for method_hash in contracts_methods[contract_address]:
abis.append(contracts_ABIs[contract_address][method_hash])
# generate interface
interfaces[contract_address] = web3_client.eth.contract(
address=web3_client.toChecksumAddress(contract_address), abi=abis
)
responces: Dict[str, Any] = {}
# reverse call_tree
call_tree_levels = sorted(calls.keys(), reverse=True)[:-1]
engine = create_moonstream_engine(
MOONSTREAM_DB_URI,
pool_pre_ping=True,
pool_size=MOONSTREAM_POOL_SIZE,
statement_timeout=MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
)
process_session = sessionmaker(bind=engine)
db_session = process_session()
# run crawling of levels
try:
# initial call of level 0 all call without subcalls directly moved there
logger.info("Crawl level: 0")
logger.info(f"Jobs amount: {len(calls[0])}")
crawl_calls_level(
db_session,
calls[0],
responces,
contracts_ABIs,
interfaces,
batch_size,
multicall_method,
block_number,
blockchain_type,
block_timestamp,
)
for level in call_tree_levels:
logger.info(f"Crawl level: {level}")
logger.info(f"Jobs amount: {len(calls[level])}")
crawl_calls_level(
db_session,
calls[level],
responces,
contracts_ABIs,
interfaces,
batch_size,
multicall_method,
block_number,
blockchain_type,
block_timestamp,
)
finally:
db_session.close()
def handle_crawl(args: argparse.Namespace) -> None:
"""
Ability to track states of the contracts.
Read all view methods of the contracts and crawl
"""
my_job = {
"type": "function",
"stateMutability": "view",
"inputs": [
{
"internalType": "uint256",
"name": "tokenId",
"type": "uint256",
"value": {
"type": "function",
"name": "totalSupply",
"outputs": [
{
"internalType": "uint256",
"name": "",
"type": "uint256",
}
],
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
"inputs": [],
},
}
],
"name": "tokenURI",
"outputs": [{"internalType": "string", "name": "", "type": "string"}],
"address": "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f",
}
blockchain_type = AvailableBlockchainType(args.blockchain)
parse_jobs(
[my_job], blockchain_type, args.block_number, args.batch_size, args.access_id
)
def parse_abi(args: argparse.Namespace) -> None:
"""
Parse the abi of the contract and save it to the database.
"""
with open(args.abi_file, "r") as f:
# read json and parse only stateMutability equal to view
abi = json.load(f)
output_json = []
for method in abi:
if method.get("stateMutability") and method["stateMutability"] == "view":
output_json.append(method)
with open(f"view+{args.abi_file}", "w") as f:
json.dump(output_json, f)
def main() -> None:
parser = argparse.ArgumentParser()
parser.set_defaults(func=lambda _: parser.print_help())
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
)
subparsers = parser.add_subparsers()
view_state_crawler_parser = subparsers.add_parser(
"crawl-jobs",
help="continuous crawling the view methods from job structure", # TODO(ANDREY): move tasks to journal
)
view_state_crawler_parser.add_argument(
"--blockchain",
"-b",
type=str,
help="Type of blovkchain wich writng in database",
required=True,
)
view_state_crawler_parser.add_argument(
"--block-number", "-N", type=str, help="Block number."
)
view_state_crawler_parser.add_argument(
"--batch-size",
"-s",
type=int,
default=500,
help="Size of chunks wich send to Multicall2 contract.",
)
view_state_crawler_parser.set_defaults(func=handle_crawl)
generate_view_parser = subparsers.add_parser(
"parse-abi",
help="Parse view methods from the abi file.",
)
generate_view_parser.add_argument(
"--abi-file",
"-a",
type=str,
help="Path to abi file.",
)
generate_view_parser.set_defaults(func=parse_abi)
args = parser.parse_args()
args.func(args)
if __name__ == "__main__":
main()

Wyświetl plik

@ -0,0 +1,61 @@
import logging
import json
from typing import Dict, Any
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
from sqlalchemy.orm import Session
from ..settings import VIEW_STATE_CRAWLER_LABEL
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def view_call_to_label(
blockchain_type: AvailableBlockchainType,
call: Dict[str, Any],
label_name=VIEW_STATE_CRAWLER_LABEL,
):
"""
Creates a label model.
"""
label_model = get_label_model(blockchain_type)
sanityzed_label_data = json.loads(
json.dumps(
{
"type": "view",
"name": call["name"],
"result": call["result"],
"inputs": call["inputs"],
"call_data": call["call_data"],
"status": call["status"],
}
).replace(r"\u0000", "")
)
label = label_model(
label=label_name,
label_data=sanityzed_label_data,
address=call["address"],
block_number=call["block_number"],
transaction_hash=None,
block_timestamp=call["block_timestamp"],
)
return label
def commit_session(db_session: Session) -> None:
"""
Save labels in the database.
"""
try:
logger.info("Committing session to database")
db_session.commit()
except Exception as e:
logger.error(f"Failed to save labels: {e}")
db_session.rollback()
raise e

Wyświetl plik

@ -0,0 +1,246 @@
import getpass
import os
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from eth_account.account import Account # type: ignore
from eth_abi import encode_single, decode_single
from eth_typing.evm import ChecksumAddress
from eth_utils import function_signature_to_4byte_selector
from hexbytes.main import HexBytes
from web3 import Web3
from web3.contract import ContractFunction
from web3.providers.ipc import IPCProvider
from web3.providers.rpc import HTTPProvider
from web3.types import ABI, Nonce, TxParams, TxReceipt, Wei
from web3._utils.abi import normalize_event_input_types
class ContractConstructor:
def __init__(self, *args: Any):
self.args = args
def build_transaction(
web3: Web3,
builder: Union[ContractFunction, Any],
sender: ChecksumAddress,
) -> Union[TxParams, Any]:
"""
Builds transaction json with the given arguments. It is not submitting transaction
Arguments:
- web3: Web3 client
- builder: ContractFunction or other class that has method buildTransaction(TxParams)
- sender: `from` value of transaction, address which is sending this transaction
- maxFeePerGas: Optional, max priority fee for dynamic fee transactions in Wei
- maxPriorityFeePerGas: Optional the part of the fee that goes to the miner
"""
transaction = builder.buildTransaction(
{
"from": sender,
"nonce": get_nonce(web3, sender),
}
)
return transaction
def get_nonce(web3: Web3, address: ChecksumAddress) -> Nonce:
"""
Returns Nonce: number of transactions for given address
"""
nonce = web3.eth.get_transaction_count(address)
return nonce
def submit_transaction(
web3: Web3, transaction: Union[TxParams, Any], signer_private_key: str
) -> HexBytes:
"""
Signs and submits json transaction to blockchain from the name of signer
"""
signed_transaction = web3.eth.account.sign_transaction(
transaction, private_key=signer_private_key
)
return submit_signed_raw_transaction(web3, signed_transaction.rawTransaction)
def submit_signed_raw_transaction(
web3: Web3, signed_raw_transaction: HexBytes
) -> HexBytes:
"""
Submits already signed raw transaction.
"""
transaction_hash = web3.eth.send_raw_transaction(signed_raw_transaction)
return transaction_hash
def wait_for_transaction_receipt(web3: Web3, transaction_hash: HexBytes):
return web3.eth.wait_for_transaction_receipt(transaction_hash)
def deploy_contract(
web3: Web3,
contract_bytecode: str,
contract_abi: List[Dict[str, Any]],
deployer: ChecksumAddress,
deployer_private_key: str,
constructor_arguments: Optional[List[Any]] = None,
) -> Tuple[HexBytes, ChecksumAddress]:
"""
Deploys smart contract to blockchain
Arguments:
- web3: web3 client
- contract_bytecode: Compiled smart contract bytecode
- contract_abi: Json abi of contract. Must include `constructor` function
- deployer: Address which is deploying contract. Deployer will pay transaction fee
- deployer_private_key: Private key of deployer. Needed for signing and submitting transaction
- constructor_arguments: arguments that are passed to `constructor` function of the smart contract
"""
contract = web3.eth.contract(abi=contract_abi, bytecode=contract_bytecode)
if constructor_arguments is None:
transaction = build_transaction(web3, contract.constructor(), deployer)
else:
transaction = build_transaction(
web3, contract.constructor(*constructor_arguments), deployer
)
transaction_hash = submit_transaction(web3, transaction, deployer_private_key)
transaction_receipt = wait_for_transaction_receipt(web3, transaction_hash)
contract_address = transaction_receipt.contractAddress
return transaction_hash, web3.toChecksumAddress(contract_address)
def deploy_contract_from_constructor_function(
web3: Web3,
contract_bytecode: str,
contract_abi: List[Dict[str, Any]],
deployer: ChecksumAddress,
deployer_private_key: str,
constructor: ContractFunction,
) -> Tuple[HexBytes, ChecksumAddress]:
"""
Deploys smart contract to blockchain from constructor ContractFunction
Arguments:
- web3: web3 client
- contract_bytecode: Compiled smart contract bytecode
- contract_abi: Json abi of contract. Must include `constructor` function
- deployer: Address which is deploying contract. Deployer will pay transaction fee
- deployer_private_key: Private key of deployer. Needed for signing and submitting transaction
- constructor:`constructor` function of the smart contract
"""
contract = web3.eth.contract(abi=contract_abi, bytecode=contract_bytecode)
transaction = build_transaction(
web3, contract.constructor(*constructor.args), deployer
)
transaction_hash = submit_transaction(web3, transaction, deployer_private_key)
transaction_receipt = wait_for_transaction_receipt(web3, transaction_hash)
contract_address = transaction_receipt.contractAddress
return transaction_hash, web3.toChecksumAddress(contract_address)
def decode_transaction_input(web3: Web3, transaction_input: str, abi: Dict[str, Any]):
contract = web3.eth.contract(abi=abi)
return contract.decode_function_input(transaction_input)
def read_keys_from_cli() -> Tuple[ChecksumAddress, str]:
private_key = getpass.getpass(prompt="Enter private key of your address:")
account = Account.from_key(private_key)
return (Web3.toChecksumAddress(account.address), private_key)
def read_keys_from_env() -> Tuple[ChecksumAddress, str]:
private_key = os.environ.get("MOONWORM_ETHEREUM_ADDRESS_PRIVATE_KEY")
if private_key is None:
raise ValueError(
"MOONWORM_ETHEREUM_ADDRESS_PRIVATE_KEY env variable is not set"
)
try:
account = Account.from_key(private_key)
return (Web3.toChecksumAddress(account.address), private_key)
except:
raise ValueError(
"Failed to initiate account from MOONWORM_ETHEREUM_ADDRESS_PRIVATE_KEY"
)
def connect(web3_uri: str) -> Web3:
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
else:
web3_provider = Web3.IPCProvider(web3_uri)
web3_client = Web3(web3_provider)
return web3_client
def read_web3_provider_from_env() -> Web3:
provider_path = os.environ.get("MOONWORM_WEB3_PROVIDER_URI")
if provider_path is None:
raise ValueError("MOONWORM_WEB3_PROVIDER_URI env variable is not set")
return connect(provider_path)
def read_web3_provider_from_cli() -> Web3:
provider_path = input("Enter web3 uri path: ")
return connect(provider_path)
def cast_to_python_type(evm_type: str) -> Callable:
if evm_type.startswith(("uint", "int")):
return int
elif evm_type.startswith("bytes"):
return bytes
elif evm_type == "string":
return str
elif evm_type == "address":
return Web3.toChecksumAddress
elif evm_type == "bool":
return bool
else:
raise ValueError(f"Cannot convert to python type {evm_type}")
class FunctionInput:
def __init__(self, name: str, value: Any, solidity_type: str):
self.name = name
self.value = value
self.solidity_type = solidity_type
class FunctionSignature:
def __init__(self, function: ContractFunction):
self.name = function.abi["name"]
self.inputs = [
{"name": arg["name"], "type": arg["type"]}
for arg in normalize_event_input_types(function.abi.get("inputs", []))
]
self.input_types_signature = "({})".format(
",".join([inp["type"] for inp in self.inputs])
)
self.output_types_signature = "({})".format(
",".join(
[
arg["type"]
for arg in normalize_event_input_types(
function.abi.get("outputs", [])
)
]
)
)
self.signature = "{}{}".format(self.name, self.input_types_signature)
self.fourbyte = function_signature_to_4byte_selector(self.signature)
def encode_data(self, args=None) -> bytes:
return (
self.fourbyte + encode_single(self.input_types_signature, args)
if args
else self.fourbyte
)
def decode_data(self, output):
return decode_single(self.output_types_signature, output)

Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.2.2"
MOONCRAWL_VERSION = "0.2.3"

Wyświetl plik

@ -11,8 +11,6 @@ const Status = () => {
const { serverListStatusCache } = useStatus();
console.log(serverListStatusCache?.data);
const moonstreamapiStatus = serverListStatusCache?.data?.filter(
(i) => i.name === "moonstream_api"
)[0];
@ -40,9 +38,6 @@ const Status = () => {
const dbReplicaServerStatus = serverListStatusCache?.data?.filter(
(i) => i.name === "moonstream_database_replica"
)[0];
const unimLeaderboardStatus = serverListStatusCache?.data?.filter(
(i) => i.name === "unim_leaderboard"
)[0];
const StatusRow = (props) => {
return (
@ -261,25 +256,6 @@ const Status = () => {
: 0}
</Text>
</StatusRow>
<br />
<StatusRow
title="Unim Leaderboard server"
cache={serverListStatusCache}
>
<Text
color={
unimLeaderboardStatus?.status_code == 200
? healthyStatusColor
: downStatusColor
}
>
{unimLeaderboardStatus?.status_code == 200
? healthyStatusText
: downStatusText}
</Text>
</StatusRow>
</chakra.span>
</>
);