mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge branch 'main' into add-batch-jobs-endpoint

pull/847/head
Andrey 2023-07-13 11:31:31 +03:00
rodzic 1a68bd7a1f 734bbaac5e
commit 36d7796baf
42 zmienionych plików z 26784 dodań i 231 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
.github/workflows/release.moonstreamdb.yml vendored
Wyświetl plik

@ -7,7 +7,7 @@ on:
defaults:
run:
working-directory: db
working-directory: moonstreamdb
jobs:
publish:

2
clients/python/README.md
Wyświetl plik

@ -82,7 +82,7 @@ Stream of event packs will be generating from recent timestamp to older and inne
**From timestamp to timestamp, from bottom to top**
When `start_time` is less then `end_time`.
When `start_time` is less than `end_time`.
```python
for events in mc.create_stream(

23
crawlers/deploy/deploy.bash
Wyświetl plik

@ -106,6 +106,11 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction
WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service"
WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer"
# ZkSync Era testnet
ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service"
ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE="zksync-era-testnet-missing.service"
set -eu
echo
@ -499,3 +504,21 @@ cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.c
cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
# ZkSync Era
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet missing service and timer with: ${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"

11
crawlers/deploy/zksync-era-testnet-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n
CPUWeight=50
SyslogIdentifier=zksync-era-testnet-missing

9
crawlers/deploy/zksync-era-testnet-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/zksync-era-testnet-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=ZkSync Era testnet block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-testnet-synchronize
[Install]
WantedBy=multi-user.target

27
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -27,6 +27,7 @@ from .settings import (
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
NB_ACCESS_ID_HEADER,
NB_DATA_SOURCE_HEADER,
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
@ -70,6 +71,8 @@ def connect(
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.WYRM:
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -123,6 +126,19 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
)
if blockchain_type == AvailableBlockchainType.XDAI:
block_obj.author = block.author
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
block_obj.mix_hash = block.get("mixHash", "")
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_batch_number = (
int(block.get("l1BatchNumber"), 0)
if block.get("l1BatchNumber") is not None
else None
)
block_obj.l1_batch_timestamp = (
int(block.get("l1BatchTimestamp"), 0)
if block.get("l1BatchTimestamp") is not None
else None
)
db_session.add(block_obj)
@ -152,6 +168,17 @@ def add_block_transactions(
transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None,
value=tx.value,
)
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
tx_obj.l1_batch_number = (
int(tx.get("l1BatchNumber"), 0)
if tx.get("l1BatchNumber") is not None
else None
)
tx_obj.l1_batch_tx_index = (
int(tx.get("l1BatchTxIndex"), 0)
if tx.get("l1BatchTxIndex") is not None
else None
)
db_session.add(tx_obj)

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -130,6 +130,8 @@ def continuous_crawler(
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

3
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -35,6 +35,7 @@ class SubscriptionTypes(Enum):
MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
XDAI_BLOCKCHAIN = "xdai_smartcontract"
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -139,6 +140,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.XDAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.WYRM:
return SubscriptionTypes.WYRM_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -68,6 +68,8 @@ def function_call_crawler(
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

8
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -114,6 +114,14 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

3
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -48,6 +48,7 @@ subscription_id_by_blockchain = {
"mumbai": "mumbai_smartcontract",
"xdai": "xdai_smartcontract",
"wyrm": "wyrm_smartcontract",
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
}
blockchain_by_subscription_id = {
@ -56,11 +57,13 @@ blockchain_by_subscription_id = {
"mumbai_blockchain": "mumbai",
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
"xdai_smartcontract": "xdai",
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
}

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.1"
MOONCRAWL_VERSION = "0.3.2"

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -25,6 +25,7 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_pol
export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.8",
"chardet",
"fastapi",
"moonstreamdb>=0.3.3",
"moonstreamdb>=0.3.4",
"moonstream>=0.1.1",
"moonstream-entity>=0.0.5",
"moonworm[moonstream]>=0.6.2",

2
docs/architecture.md
Wyświetl plik

@ -49,7 +49,7 @@ the schema for this Postgres database as well as migrations that you can use to
database yourself.
The [`db/`](../db/) directory contains:
1. A Python package called `moonstreamdb` which defines the databse schema and can be used as a
1. A Python package called `moonstreamdb` which defines the database schema and can be used as a
Python library to interact with the data store.
2. [Alembic](https://alembic.sqlalchemy.org/en/latest/) migrations which can be used via the
[`alembic.sh`](../db/alembic.sh) shell script to run the migrations against a Postgres database

2
moonstreamapi/moonstreamapi/actions.py
Wyświetl plik

@ -59,11 +59,13 @@ blockchain_by_subscription_id = {
"mumbai_blockchain": "mumbai",
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
"xdai_smartcontract": "xdai",
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
}

8
moonstreamapi/moonstreamapi/admin/cli.py
Wyświetl plik

@ -478,18 +478,12 @@ This CLI is configured to work with the following API URLs:
parser_moonworm_tasks_add.set_defaults(func=moonworm_tasks_add_subscription_handler)
queries_parser = subcommands.add_parser(
"queries", description="Manage Moonstream queries"
)
queries_parser.set_defaults(func=lambda _: queries_parser.print_help())
queries_subcommands = queries_parser.add_subparsers(
description="Query commands"
)
queries_subcommands = queries_parser.add_subparsers(description="Query commands")
create_query_parser = queries_subcommands.add_parser(
"create-template", description="Create query template"

11
moonstreamapi/moonstreamapi/admin/queries.py
Wyświetl plik

@ -24,12 +24,11 @@ from ..actions import get_all_entries_from_search, name_normalization
logger = logging.getLogger(__name__)
def create_query_template(args: argparse.Namespace) -> None:
"""
Create query template for all queries resources.
"""
query = ""
with args.query_file:
query = textwrap.indent(args.query_file.read(), " ")
@ -39,7 +38,6 @@ def create_query_template(args: argparse.Namespace) -> None:
name = f"template_{name_normalization(args.name)}"
try:
entry = bc.create_entry(
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
title=args.name,
@ -51,18 +49,17 @@ def create_query_template(args: argparse.Namespace) -> None:
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
except BugoutResponseException as err:
logger.error(f"Failed to create query template: {err}")
return
except Exception as err:
logger.error(f"Failed to create query template: {err}")
return
logger.info(f"Query template created: {entry.id}")
logger.info(f"Query template created url name: {name}")
### Add query id
try:
@ -81,4 +78,4 @@ def create_query_template(args: argparse.Namespace) -> None:
logger.error(f"Failed to add query id: {err}")
return
logger.info(f"Query created: {json.dumps(entry.dict(), indent=4)}")
logger.info(f"Query created: {json.dumps(entry.dict(), indent=4)}")

22
moonstreamapi/moonstreamapi/admin/subscription_types.py
Wyświetl plik

@ -72,6 +72,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"zksync_era_testnet_smartcontract": SubscriptionTypeResourceData(
id="zksync_era_testnet_smartcontract",
name="zkSync Era testnet smartcontract",
blockchain="zksync_era_testnet",
choices=["input:address", "tag:erc721"],
description="Contracts events and tx_calls of contract of zkSync Era testnet blockchain.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-testnet-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_blockchain": SubscriptionTypeResourceData(
id="ethereum_blockchain",
name="Ethereum transactions",
@ -127,6 +138,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=False,
),
"zksync_era_testnet_blockchain": SubscriptionTypeResourceData(
id="zksync_era_testnet_blockchain",
name="zkSync Era testnet transactions",
blockchain="zksync_era_testnet",
choices=["input:address", "tag:erc721"],
description="ZkSync Era testnet chain transactions subscription.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-testnet-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=False,
),
"ethereum_whalewatch": SubscriptionTypeResourceData(
id="ethereum_whalewatch",
name="Ethereum whale watch",

43
moonstreamapi/moonstreamapi/data.py
Wyświetl plik

@ -2,14 +2,17 @@
Pydantic schemas for the Moonstream HTTP API
"""
from datetime import datetime
import json
from enum import Enum
from typing import Any, Dict, List, Optional, Union, Literal
from uuid import UUID
from xmlrpc.client import Boolean
from fastapi import Form
from pydantic import BaseModel, Field, validator
from sqlalchemy import false
USER_ONBOARDING_STATE = "onboarding_state"
BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver"
@ -47,6 +50,8 @@ class SubscriptionResourceData(BaseModel):
abi: Optional[str]
color: Optional[str]
label: Optional[str]
description: Optional[str] = None
tags: List[str] = Field(default_factory=list)
user_id: str
subscription_type_id: Optional[str]
jobs_status: Optional[List[Dict[str, Any]]] = Field(default_factory=list)
@ -244,6 +249,40 @@ class SubdcriptionsAbiResponse(BaseModel):
abi: str
class UpdateSubscriptionRequest(BaseModel):
color: Optional[str] = Form(None)
label: Optional[str] = Form(None)
abi: Optional[str] = Form(None)
description: Optional[str] = Form(None)
tags: Optional[List[Dict[str, str]]] = Form(None)
@validator("tags", pre=True, always=True)
def transform_to_dict(cls, v):
if isinstance(v, str):
return json.loads(v)
elif isinstance(v, list):
return v
return []
class CreateSubscriptionRequest(BaseModel):
address: str = Form(...)
subscription_type_id: str = Form(...)
color: str = Form(...)
label: str = Form(...)
abi: Optional[str] = Form(None)
description: Optional[str] = Form(None)
tags: Optional[List[Dict[str, str]]] = Form(None)
@validator("tags", pre=True, always=True)
def transform_to_dict(cls, v):
if isinstance(v, str):
return json.loads(v)
elif isinstance(v, list):
return v
return []
class DashboardMeta(BaseModel):
subscription_id: UUID
generic: Optional[List[Dict[str, str]]]
@ -296,8 +335,8 @@ class QueryInfoResponse(BaseModel):
preapprove: bool = False
approved: bool = False
parameters: Dict[str, Any] = Field(default_factory=dict)
created_at: Optional[datetime]
updated_at: Optional[datetime]
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
class SuggestedQueriesResponse(BaseModel):

2
moonstreamapi/moonstreamapi/providers/__init__.py
Wyświetl plik

@ -51,10 +51,12 @@ event_providers: Dict[str, Any] = {
moonworm_provider.PolygonMoonwormProvider.event_type: moonworm_provider.PolygonMoonwormProvider,
moonworm_provider.MumbaiMoonwormProvider.event_type: moonworm_provider.MumbaiMoonwormProvider,
moonworm_provider.XDaiMoonwormProvider.event_type: moonworm_provider.XDaiMoonwormProvider,
moonworm_provider.ZkSyncEraTestnetMoonwormProvider.event_type: moonworm_provider.ZkSyncEraTestnetMoonwormProvider,
transactions.EthereumTransactions.event_type: transactions.EthereumTransactions,
transactions.PolygonTransactions.event_type: transactions.PolygonTransactions,
transactions.MumbaiTransactions.event_type: transactions.MumbaiTransactions,
transactions.XDaiTransactions.event_type: transactions.XDaiTransactions,
transactions.ZkSyncEraTestnetTransactions.event_type: transactions.ZkSyncEraTestnetTransactions,
bugout.polygon_whalewatch_provider.event_type: bugout.polygon_whalewatch_provider,
bugout.ethereum_txpool_provider.event_type: bugout.ethereum_txpool_provider,
bugout.ethereum_whalewatch_provider.event_type: bugout.ethereum_whalewatch_provider,

8
moonstreamapi/moonstreamapi/providers/moonworm_provider.py
Wyświetl plik

@ -21,6 +21,7 @@ ethereum_event_type = "ethereum_blockchain"
polygon_event_type = "polygon_blockchain"
mumbai_event_type = "mumbai_blockchain"
xdai_event_type = "xdai_blockchain"
zksync_era_testnet_event_type = "zksync_era_testnet_blockchain"
allowed_tags = ["tag:erc721"]
description = f"""Event provider for transactions from the Ethereum blockchain.
@ -413,3 +414,10 @@ XDaiMoonwormProvider = MoonwormProvider(
description="Provider for reviving transactions from XDai tables.",
streamboaundary_range_limit=2 * 60 * 60,
)
ZkSyncEraTestnetMoonwormProvider = MoonwormProvider(
event_type="zksync_era_testnet_smartcontract",
blockchain=AvailableBlockchainType("zksync_era_testnet"),
description="Provider for reviving transactions from zkSync Era testnet tables.",
streamboaundary_range_limit=2 * 60 * 60,
)

7
moonstreamapi/moonstreamapi/providers/transactions.py
Wyświetl plik

@ -475,3 +475,10 @@ XDaiTransactions = TransactionsProvider(
description="Provider for resiving transactions from XDai tables.",
streamboaundary_range_limit=2 * 60 * 60,
)
ZkSyncEraTestnetTransactions = TransactionsProvider(
event_type="zksync_era_testnet_blockchain",
blockchain=AvailableBlockchainType("zksync_era_testnet"),
description="Provider for resiving transactions from ZkSync Era testnet tables.",
streamboaundary_range_limit=2 * 60 * 60,
)

16
moonstreamapi/moonstreamapi/routes/queries.py
Wyświetl plik

@ -7,7 +7,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union
from uuid import UUID
from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry
from bugout.exceptions import BugoutResponseException
from fastapi import APIRouter, Body, Request
@ -157,7 +156,6 @@ async def create_query_handler(
return entry
@router.get("/templates", tags=["queries"])
def get_suggested_queries(
supported_interfaces: Optional[List[str]] = None,
@ -223,7 +221,6 @@ async def get_query_handler(
) -> data.QueryInfoResponse:
token = request.state.token
# normalize query name
try:
@ -234,7 +231,6 @@ async def get_query_handler(
detail=f"Provided query name can't be normalize please select different.",
)
# check in templates
try:
@ -283,7 +279,6 @@ async def get_query_handler(
else:
query_id = entries.results[0].entry_url.split("/")[-1]
entry = entries.results[0]
try:
@ -312,7 +307,6 @@ async def get_query_handler(
else:
query_parameters[param] = None
print(type(entry.created_at))
return data.QueryInfoResponse(
@ -395,7 +389,6 @@ async def update_query_data_handler(
detail=f"Provided query name can't be normalize please select different.",
)
# check in templates
try:
@ -510,9 +503,7 @@ async def get_access_link_handler(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
query=f"tag:query_template tag:query_url:{query_name_normalized}",
filters=[
f"context_type:{MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE}"
],
filters=[f"context_type:{MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE}"],
limit=1,
)
except BugoutResponseException as e:
@ -522,7 +513,6 @@ async def get_access_link_handler(
raise MoonstreamHTTPException(status_code=500, internal_error=e)
if len(entries.results) == 0:
try:
query_id = get_query_by_name(query_name, token)
except NameNormalizationException:
@ -532,7 +522,6 @@ async def get_access_link_handler(
)
try:
entries = bc.search(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
@ -552,7 +541,6 @@ async def get_access_link_handler(
)
try:
s3_response = None
if entries.results[0].content:
@ -631,4 +619,4 @@ async def remove_query_handler(
except Exception as e:
raise MoonstreamHTTPException(status_code=500, internal_error=e)
return entry
return entry

141
moonstreamapi/moonstreamapi/routes/subscriptions.py
Wyświetl plik

@ -6,7 +6,6 @@ import hashlib
import json
import logging
from typing import Any, Dict, List, Optional
import traceback
from bugout.exceptions import BugoutResponseException
from bugout.data import BugoutSearchResult
@ -29,7 +28,11 @@ from ..admin import subscription_types
from ..middleware import MoonstreamHTTPException
from ..reporter import reporter
from ..settings import bugout_client as bc, entity_client as ec
from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, THREAD_TIMEOUT_SECONDS
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_ENTITIES_RESERVED_TAGS,
THREAD_TIMEOUT_SECONDS,
)
from ..web3_provider import (
yield_web3_provider,
)
@ -49,11 +52,6 @@ BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION = "entity_subscription"
async def add_subscription_handler(
request: Request,
background_tasks: BackgroundTasks,
address: str = Form(...),
color: str = Form(...),
label: str = Form(...),
subscription_type_id: str = Form(...),
abi: Optional[str] = Form(None),
web3: Web3 = Depends(yield_web3_provider),
) -> data.SubscriptionResourceData:
"""
@ -61,6 +59,21 @@ async def add_subscription_handler(
"""
token = request.state.token
form = await request.form()
try:
form_data = data.CreateSubscriptionRequest(**form)
except Exception as e:
raise MoonstreamHTTPException(status_code=400, detail=str(e))
address = form_data.address
color = form_data.color
label = form_data.label
abi = form_data.abi
description = form_data.description
tags = form_data.tags
subscription_type_id = form_data.subscription_type_id
if subscription_type_id != "ethereum_whalewatch":
try:
address = web3.toChecksumAddress(address)
@ -124,6 +137,28 @@ async def add_subscription_handler(
address,
)
if description:
content["description"] = description
allowed_required_fields = []
if tags:
allowed_required_fields = [
item
for item in tags
if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
]
required_fields = [
{"type": "subscription"},
{"subscription_type_id": f"{subscription_type_id}"},
{"color": f"{color}"},
{"label": f"{label}"},
{"user_id": f"{user.id}"},
]
if allowed_required_fields:
required_fields.extend(allowed_required_fields)
try:
collection_id = get_entity_subscription_collection_id(
resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
@ -140,13 +175,7 @@ async def add_subscription_handler(
subscription_type_id
].blockchain,
name=label,
required_fields=[
{"type": "subscription"},
{"subscription_type_id": f"{subscription_type_id}"},
{"color": f"{color}"},
{"label": f"{label}"},
{"user_id": f"{user.id}"},
],
required_fields=required_fields,
secondary_fields=content,
)
except EntityCollectionNotFoundException as e:
@ -163,6 +192,13 @@ async def add_subscription_handler(
detail="Currently unable to get collection id",
)
normalized_entity_tags = [
f"{key}:{value}"
for tag in entity.required_fields
for key, value in tag.items()
if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
]
return data.SubscriptionResourceData(
id=str(entity.entity_id),
user_id=str(user.id),
@ -170,6 +206,8 @@ async def add_subscription_handler(
color=color,
label=label,
abi=entity.secondary_fields.get("abi"),
description=entity.secondary_fields.get("description"),
tags=normalized_entity_tags,
subscription_type_id=subscription_type_id,
updated_at=entity.updated_at,
created_at=entity.created_at,
@ -240,6 +278,8 @@ async def delete_subscription_handler(request: Request, subscription_id: str):
color=color,
label=label,
abi=abi,
description=deleted_entity.secondary_fields.get("description"),
tags=deleted_entity.required_fields,
subscription_type_id=subscription_type_id,
updated_at=deleted_entity.updated_at,
created_at=deleted_entity.created_at,
@ -303,6 +343,13 @@ async def get_subscriptions_handler(
if "label" in tag:
label = tag["label"]
normalized_entity_tags = [
f"{key}:{value}"
for tag in tags
for key, value in tag.items()
if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
]
subscriptions.append(
data.SubscriptionResourceData(
id=str(subscription.entity_id),
@ -311,6 +358,8 @@ async def get_subscriptions_handler(
color=color,
label=label,
abi=subscription.secondary_fields.get("abi", None),
description=subscription.secondary_fields.get("description"),
tags=normalized_entity_tags,
subscription_type_id=subscription_type_id,
updated_at=subscription.updated_at,
created_at=subscription.created_at,
@ -335,9 +384,6 @@ async def update_subscriptions_handler(
request: Request,
subscription_id: str,
background_tasks: BackgroundTasks,
color: Optional[str] = Form(None),
label: Optional[str] = Form(None),
abi: Optional[str] = Form(None),
) -> data.SubscriptionResourceData:
"""
Get user's subscriptions.
@ -346,9 +392,17 @@ async def update_subscriptions_handler(
user = request.state.user
update_required_fields = []
form = await request.form()
try:
form_data = data.UpdateSubscriptionRequest(**form)
except Exception as e:
raise MoonstreamHTTPException(status_code=400, detail=str(e))
update_secondary_fields = {}
color = form_data.color
label = form_data.label
abi = form_data.abi
description = form_data.description
tags = form_data.tags
try:
collection_id = get_entity_subscription_collection_id(
@ -366,7 +420,13 @@ async def update_subscriptions_handler(
subscription_type_id = None
update_required_fields = subscription_entity.required_fields
update_required_fields = [
field
for field in subscription_entity.required_fields
if any(key in field for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
]
update_secondary_fields = subscription_entity.secondary_fields
for field in update_required_fields:
if "subscription_type_id" in field:
@ -377,7 +437,7 @@ async def update_subscriptions_handler(
f"Subscription entity {subscription_id} in collection {collection_id} has no subscription_type_id malformed subscription entity"
)
raise MoonstreamHTTPException(
status_code=404,
status_code=409,
detail="Not valid subscription entity",
)
@ -394,13 +454,19 @@ async def update_subscriptions_handler(
raise MoonstreamHTTPException(status_code=500, internal_error=e)
for field in update_required_fields:
if "color" in field and color is not None:
field["color"] = color
if "color" in field:
if color is not None:
field["color"] = color
else:
color = field["color"]
if "label" in field and label is not None:
field["label"] = label
if "label" in field:
if label is not None:
field["label"] = label
else:
label = field["label"]
if abi:
if abi is not None:
try:
json_abi = json.loads(abi)
except json.JSONDecodeError:
@ -414,9 +480,19 @@ async def update_subscriptions_handler(
hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest()
update_secondary_fields["abi_hash"] = hash
else:
update_secondary_fields = subscription_entity.secondary_fields
if description is not None:
update_secondary_fields["description"] = description
if tags:
allowed_required_fields = [
item
for item in tags
if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
]
if allowed_required_fields:
update_required_fields.extend(allowed_required_fields)
try:
subscription = ec.update_entity(
token=token,
@ -441,6 +517,13 @@ async def update_subscriptions_handler(
subscription.address,
)
normalized_entity_tags = [
f"{key}:{value}"
for tag in subscription.required_fields
for key, value in tag.items()
if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
]
return data.SubscriptionResourceData(
id=str(subscription.entity_id),
user_id=str(user.id),
@ -448,6 +531,8 @@ async def update_subscriptions_handler(
color=color,
label=label,
abi=subscription.secondary_fields.get("abi"),
description=subscription.secondary_fields.get("description"),
tags=normalized_entity_tags,
subscription_type_id=subscription_type_id,
updated_at=subscription_entity.updated_at,
created_at=subscription_entity.created_at,

26191
moonstreamapi/moonstreamapi/selectors_storage.py
Wyświetl plik

File diff suppressed because one or more lines are too long

17
moonstreamapi/moonstreamapi/settings.py
Wyświetl plik

@ -137,6 +137,13 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
if MOONSTREAM_S3_QUERIES_BUCKET == "":
@ -150,6 +157,16 @@ if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "":
"MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set"
)
# Entities reserved tags
MOONSTREAM_ENTITIES_RESERVED_TAGS = [
"type",
"subscription_type_id",
"color",
"label",
"user_id",
"address",
"blockchain",
]
## Moonstream resources types

2
moonstreamapi/moonstreamapi/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAMAPI_VERSION = "0.2.6"
MOONSTREAMAPI_VERSION = "0.2.7"

3
moonstreamapi/moonstreamapi/web3_provider.py
Wyświetl plik

@ -19,6 +19,7 @@ from .settings import (
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
multicall_contracts,
multicall_contract_abi,
)
@ -69,6 +70,8 @@ def connect(
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.WYRM:
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")

2
moonstreamapi/requirements.txt
Wyświetl plik

@ -37,7 +37,7 @@ lru-dict==1.1.8
Mako==1.2.3
MarkupSafe==2.1.1
moonstream-entity==0.0.5
moonstreamdb==0.3.3
moonstreamdb==0.3.4
multiaddr==0.0.9
multidict==6.0.2
netaddr==0.8.0

2
moonstreamapi/setup.py
Wyświetl plik

@ -16,7 +16,7 @@ setup(
"bugout>=0.2.9",
"moonstream-entity>=0.0.5",
"fastapi",
"moonstreamdb>=0.3.3",
"moonstreamdb>=0.3.4",
"humbug",
"pydantic",
"pyevmasm",

3
moonstreamdb/.isort.cfg 100644
Wyświetl plik

@ -0,0 +1,3 @@
[settings]
profile = black
multi_line_output = 3

41
moonstreamdb/alembic/env.py
Wyświetl plik

@ -1,7 +1,6 @@
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from sqlalchemy import engine_from_config, pool
from alembic import context
@ -26,18 +25,27 @@ target_metadata = MoonstreamBase.metadata
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
from moonstreamdb.models import (
EthereumBlock,
EthereumTransaction,
EthereumLabel,
PolygonBlock,
PolygonTransaction,
PolygonLabel,
MumbaiBlock,
MumbaiTransaction,
MumbaiLabel,
ESDFunctionSignature,
ESDEventSignature,
ESDFunctionSignature,
EthereumBlock,
EthereumLabel,
EthereumTransaction,
MumbaiBlock,
MumbaiLabel,
MumbaiTransaction,
OpenSeaCrawlingState,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
WyrmBlock,
WyrmLabel,
WyrmTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
@ -55,6 +63,15 @@ def include_symbol(tablename, schema):
ESDFunctionSignature.__tablename__,
ESDEventSignature.__tablename__,
OpenSeaCrawlingState.__tablename__,
WyrmBlock.__tablename__,
WyrmLabel.__tablename__,
WyrmTransaction.__tablename__,
XDaiBlock.__tablename__,
XDaiLabel.__tablename__,
XDaiTransaction.__tablename__,
ZkSyncEraTestnetBlock.__tablename__,
ZkSyncEraTestnetLabel.__tablename__,
ZkSyncEraTestnetTransaction.__tablename__,
}

122
moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py 100644
Wyświetl plik

@ -0,0 +1,122 @@
"""ZkSync Era model
Revision ID: e4a796c0407d
Revises: c413d5265f76
Create Date: 2023-07-12 12:27:12.814892
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'e4a796c0407d'
down_revision = 'c413d5265f76'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('zksync_era_testnet_blocks',
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('difficulty', sa.BigInteger(), nullable=True),
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
sa.Column('gas_used', sa.BigInteger(), nullable=True),
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('size', sa.Integer(), nullable=True),
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('timestamp', sa.BigInteger(), nullable=True),
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
sa.Column('l1_batch_timestamp', sa.BigInteger(), nullable=True),
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_zksync_era_testnet_blocks'))
)
op.create_index(op.f('ix_zksync_era_testnet_blocks_block_number'), 'zksync_era_testnet_blocks', ['block_number'], unique=True)
op.create_index(op.f('ix_zksync_era_testnet_blocks_hash'), 'zksync_era_testnet_blocks', ['hash'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), 'zksync_era_testnet_blocks', ['timestamp'], unique=False)
op.create_table('zksync_era_testnet_labels',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=True),
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
sa.Column('log_index', sa.Integer(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_zksync_era_testnet_labels')),
sa.UniqueConstraint('id', name=op.f('uq_zksync_era_testnet_labels_id'))
)
op.create_index(op.f('ix_zksync_era_testnet_labels_address'), 'zksync_era_testnet_labels', ['address'], unique=False)
op.create_index('ix_zksync_era_testnet_labels_address_block_number', 'zksync_era_testnet_labels', ['address', 'block_number'], unique=False)
op.create_index('ix_zksync_era_testnet_labels_address_block_timestamp', 'zksync_era_testnet_labels', ['address', 'block_timestamp'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_block_number'), 'zksync_era_testnet_labels', ['block_number'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), 'zksync_era_testnet_labels', ['block_timestamp'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_label'), 'zksync_era_testnet_labels', ['label'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), 'zksync_era_testnet_labels', ['transaction_hash'], unique=False)
op.create_table('zksync_era_testnet_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('transaction_type', sa.Integer(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
sa.Column('l1_batch_tx_index', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['block_number'], ['zksync_era_testnet_blocks.block_number'], name=op.f('fk_zksync_era_testnet_transactions_block_number_zksync_era_testnet_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_zksync_era_testnet_transactions'))
)
op.create_index(op.f('ix_zksync_era_testnet_transactions_block_number'), 'zksync_era_testnet_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_from_address'), 'zksync_era_testnet_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_gas'), 'zksync_era_testnet_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), 'zksync_era_testnet_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_hash'), 'zksync_era_testnet_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_zksync_era_testnet_transactions_to_address'), 'zksync_era_testnet_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_value'), 'zksync_era_testnet_transactions', ['value'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_zksync_era_testnet_transactions_value'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_to_address'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_hash'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_from_address'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_block_number'), table_name='zksync_era_testnet_transactions')
op.drop_table('zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_label'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_block_number'), table_name='zksync_era_testnet_labels')
op.drop_index('ix_zksync_era_testnet_labels_address_block_timestamp', table_name='zksync_era_testnet_labels')
op.drop_index('ix_zksync_era_testnet_labels_address_block_number', table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_address'), table_name='zksync_era_testnet_labels')
op.drop_table('zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), table_name='zksync_era_testnet_blocks')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_hash'), table_name='zksync_era_testnet_blocks')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_block_number'), table_name='zksync_era_testnet_blocks')
op.drop_table('zksync_era_testnet_blocks')
# ### end Alembic commands ###

72
moonstreamdb/moonstreamdb/blockchain.py
Wyświetl plik

@ -1,26 +1,27 @@
from .db import yield_db_session, yield_db_session_ctx
from enum import Enum
from typing import Type, Union
from .models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
MumbaiBlock,
MumbaiLabel,
MumbaiTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
WyrmBlock,
WyrmLabel,
WyrmTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
from enum import Enum
from typing import Type, Union
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
@ -28,17 +29,34 @@ class AvailableBlockchainType(Enum):
MUMBAI = "mumbai"
XDAI = "xdai"
WYRM = "wyrm"
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
def get_block_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]]:
) -> Type[
Union[
EthereumBlock,
PolygonBlock,
MumbaiBlock,
XDaiBlock,
WyrmBlock,
ZkSyncEraTestnetBlock,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
set proper blocks model.
"""
block_model: Type[
Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]
Union[
EthereumBlock,
PolygonBlock,
MumbaiBlock,
XDaiBlock,
WyrmBlock,
ZkSyncEraTestnetBlock,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
block_model = EthereumBlock
@ -50,6 +68,8 @@ def get_block_model(
block_model = XDaiBlock
elif blockchain_type == AvailableBlockchainType.WYRM:
block_model = WyrmBlock
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
block_model = ZkSyncEraTestnetBlock
else:
raise Exception("Unsupported blockchain type provided")
@ -58,13 +78,29 @@ def get_block_model(
def get_label_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]]:
) -> Type[
Union[
EthereumLabel,
PolygonLabel,
MumbaiLabel,
XDaiLabel,
WyrmLabel,
ZkSyncEraTestnetLabel,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
set proper block label model.
"""
label_model: Type[
Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]
Union[
EthereumLabel,
PolygonLabel,
MumbaiLabel,
XDaiLabel,
WyrmLabel,
ZkSyncEraTestnetLabel,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
label_model = EthereumLabel
@ -76,6 +112,8 @@ def get_label_model(
label_model = XDaiLabel
elif blockchain_type == AvailableBlockchainType.WYRM:
label_model = WyrmLabel
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
label_model = ZkSyncEraTestnetLabel
else:
raise Exception("Unsupported blockchain type provided")
@ -91,6 +129,7 @@ def get_transaction_model(
MumbaiTransaction,
XDaiTransaction,
WyrmTransaction,
ZkSyncEraTestnetTransaction,
]
]:
"""
@ -104,6 +143,7 @@ def get_transaction_model(
MumbaiTransaction,
XDaiTransaction,
WyrmTransaction,
ZkSyncEraTestnetTransaction,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -116,6 +156,8 @@ def get_transaction_model(
transaction_model = XDaiTransaction
elif blockchain_type == AvailableBlockchainType.WYRM:
transaction_model = WyrmTransaction
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
transaction_model = ZkSyncEraTestnetTransaction
else:
raise Exception("Unsupported blockchain type provided")

4
moonstreamdb/moonstreamdb/db.py
Wyświetl plik

@ -1,12 +1,12 @@
"""
Moonstream database connection.
"""
from contextlib import contextmanager
import os
from contextlib import contextmanager
from typing import Generator
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from sqlalchemy.orm import Session, sessionmaker
MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI")
if MOONSTREAM_DB_URI is None:

139
moonstreamdb/moonstreamdb/models.py
Wyświetl plik

@ -1,21 +1,21 @@
import uuid
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import (
VARCHAR,
BigInteger,
Column,
DateTime,
ForeignKey,
Index,
Integer,
ForeignKey,
MetaData,
Numeric,
Text,
VARCHAR,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.sql import expression
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import expression
"""
Naming conventions doc
@ -614,6 +614,137 @@ class WyrmLabel(Base): # type: ignore
)
class ZkSyncEraTestnetBlock(Base): # type: ignore
__tablename__ = "zksync_era_testnet_blocks"
block_number = Column(
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
parent_hash = Column(VARCHAR(256))
receipt_root = Column(VARCHAR(256))
uncles = Column(VARCHAR(256))
size = Column(Integer)
state_root = Column(VARCHAR(256))
timestamp = Column(BigInteger, index=True)
total_difficulty = Column(VARCHAR(256))
transactions_root = Column(VARCHAR(256))
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
mix_hash = Column(VARCHAR(256), nullable=True)
sha3_uncles = Column(VARCHAR(256), nullable=True)
l1_batch_number = Column(BigInteger, nullable=True)
l1_batch_timestamp = Column(BigInteger, nullable=True)
class ZkSyncEraTestnetTransaction(Base): # type: ignore
__tablename__ = "zksync_era_testnet_transactions"
hash = Column(
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("zksync_era_testnet_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
transaction_type = Column(Integer, nullable=True)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
l1_batch_number = Column(BigInteger, nullable=True)
l1_batch_tx_index = Column(BigInteger, nullable=True)
class ZkSyncEraTestnetLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "zksync_era_testnet_labels"
__table_args__ = (
Index(
"ix_zksync_era_testnet_labels_address_block_number",
"address",
"block_number",
unique=False,
),
Index(
"ix_zksync_era_testnet_labels_address_block_timestamp",
"address",
"block_timestamp",
unique=False,
),
)
id = Column(
UUID(as_uuid=True),
primary_key=True,
default=uuid.uuid4,
unique=True,
nullable=False,
)
label = Column(VARCHAR(256), nullable=False, index=True)
block_number = Column(
BigInteger,
nullable=True,
index=True,
)
address = Column(
VARCHAR(256),
nullable=True,
index=True,
)
transaction_hash = Column(
VARCHAR(256),
nullable=True,
index=True,
)
label_data = Column(JSONB, nullable=True)
block_timestamp = Column(BigInteger, index=True)
log_index = Column(Integer, nullable=True)
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class ESDFunctionSignature(Base): # type: ignore
"""
Function signature from blockchain (Ethereum/Polygon) Signature Database.

10
moonstreamdb/moonstreamdb/networks.py
Wyświetl plik

@ -18,6 +18,9 @@ from .models import (
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
@ -27,6 +30,7 @@ class Network(Enum):
mumbai = "mumbai"
xdai = "xdai"
wyrm = "wyrm"
zksync_era_testnet = "zksync_era_testnet"
tx_raw_types = Union[
@ -35,6 +39,7 @@ tx_raw_types = Union[
PolygonTransaction,
WyrmTransaction,
XDaiTransaction,
ZkSyncEraTestnetTransaction,
]
MODELS: Dict[Network, Dict[str, Base]] = {
@ -63,4 +68,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
"labels": WyrmLabel,
"transactions": WyrmTransaction,
},
Network.zksync_era_testnet: {
"blocks": ZkSyncEraTestnetBlock,
"labels": ZkSyncEraTestnetLabel,
"transactions": ZkSyncEraTestnetTransaction,
},
}

2
moonstreamdb/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.3"
MOONSTREAMDB_VERSION = "0.3.4"

8
nodebalancer/README.md
Wyświetl plik

@ -2,7 +2,7 @@
## Installation
- Prepare environment variables, according with `sample.env`.
- Prepare environment variables, according to `sample.env`.
- Build application
```bash
@ -60,14 +60,14 @@ This command will return a list of bugout resources of registered users to acces
]
```
`access_id` - token which allow access to nodebalancer, could be specified in both ways:
`access_id` - token which allows access to nodebalancer, could be specified in both ways:
- as a header `x-moonstream-access-id` with value `access_id`
- as query parameter `access_id=access_id`
`blockchain_access` - boolean which allow you or not to have access to blockchain node, otherwise you will be redirected to database
`blockchain_access` - boolean which allows you or not to have access to blockchain node, otherwise you will be redirected to database
`extended_methods` - boolean which allow you to call not whitelisted method to blockchain node, by default for new user this is equal to `false`
`extended_methods` - boolean which allows you to call not whitelisted method to blockchain node, by default for new user this is equal to `false`
### server

21
nodebalancer/cmd/nodebalancer/blockchain.go
Wyświetl plik

@ -36,6 +36,27 @@ var (
"net_version": true,
"web3_clientVersion": true,
// zksync methods
"zks_estimateFee": true,
"zks_estimateGasL1ToL2": true,
"zks_getAllAccountBalances": true,
"zks_getBlockDetails": true,
"zks_getBridgeContracts": true,
"zks_getBytecodeByHash": true,
"zks_getConfirmedTokens": true,
"zks_getL1BatchBlockRange": true,
"zks_getL1BatchDetails": true,
"zks_getL2ToL1LogProof": true,
"zks_getL2ToL1MsgProof": true,
"zks_getMainContract": true,
"zks_getRawBlockTransactions": true,
"zks_getTestnetPaymaster": true,
"zks_getTokenPrice": true,
"zks_getTransactionDetails": true,
"zks_L1BatchNumber": true,
"zks_L1ChainId": true,
}
)