From 79eaed49f8a8484a4a80781e492f8dca6cbf27df Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 00:55:20 +0200 Subject: [PATCH 01/55] Add query task sending. --- backend/moonstreamapi/routes/queries.py | 63 +++++++++++++++++++++++++ backend/moonstreamapi/routes/streams.py | 8 +--- 2 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 backend/moonstreamapi/routes/queries.py diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py new file mode 100644 index 00000000..edcdd724 --- /dev/null +++ b/backend/moonstreamapi/routes/queries.py @@ -0,0 +1,63 @@ +""" +The Moonstream subscriptions HTTP API +""" +import logging +from typing import Any, Dict, List, Optional + +from bugout.data import BugoutResource +from fastapi import APIRouter, Depends, Query, Request +import requests +from sqlalchemy.orm import Session + +from moonstreamdb import db + +from ..middleware import MoonstreamHTTPException +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_QUERIES_JOURNAL_ID, + MOONSTREAM_CRAWLERS_SERVER_URL, + MOONSTREAM_CRAWLERS_SERVER_PORT, +) +from ..settings import bugout_client as bc + + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/queries",) + + +@router.post("/{query_id}/update", tags=["queries"]) +async def update_query_data_handler( + request: Request, query_id: str = Query(...), +) -> Optional[Dict[str, Any]]: + """ + Request update data on S3 bucket + """ + + token = request.state.token + + try: + entries = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + query=f"#approved #query:{query_id} #user_token:{token}", + limit=1, + ) + + if entries.results and entries.results[0].content: + content = entries.results[0].content + + responce = requests.post( + f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/query_update", + json=content, + ) + + if responce.status_code != 200: + raise MoonstreamHTTPException( + status_code=responce.status_code, + detail="Task for start generate stats failed.", + ) + + return responce.json() + except: + return None diff --git a/backend/moonstreamapi/routes/streams.py b/backend/moonstreamapi/routes/streams.py index 1917b6ea..476a8b76 100644 --- a/backend/moonstreamapi/routes/streams.py +++ b/backend/moonstreamapi/routes/streams.py @@ -30,9 +30,7 @@ from .subscriptions import BUGOUT_RESOURCE_TYPE_SUBSCRIPTION logger = logging.getLogger(__name__) -router = APIRouter( - prefix="/streams", -) +router = APIRouter(prefix="/streams",) def get_user_subscriptions(token: str) -> Dict[str, List[BugoutResource]]: @@ -41,9 +39,7 @@ def get_user_subscriptions(token: str) -> Dict[str, List[BugoutResource]]: """ response = bc.list_resources( token=token, - params={ - "type": BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, - }, + params={"type": BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,}, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) From bc45e5cc76bb40790051a0b9cba54bec473e4c2c Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:29:12 +0200 Subject: [PATCH 02/55] Add init queries drone api. --- crawlers/mooncrawl/mooncrawl/api.py | 39 ++++++++++++++++--- .../mooncrawl/stats_worker/quereis.py | 35 +++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index a6c50cf2..49e09436 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -24,7 +24,7 @@ from .settings import ( MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from .version import MOONCRAWL_VERSION -from .stats_worker import dashboard +from .stats_worker import dashboard, queries logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -80,17 +80,14 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, - background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, - resource_id=stats_update.dashboard_id, - timeout=10, + token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, ) # get all user subscriptions @@ -166,3 +163,33 @@ async def status_handler( ) return presigned_urls_response + + +@app.post("/jobs/{query_id}/query_update", tags=["jobs"]) +async def queries_data_update_handler( + query_id: str, query: Any, background_tasks: BackgroundTasks, +) -> str: + + s3_client = boto3.client("s3") + + try: + + background_tasks.add_task( + queries.data_generate, + bucket="queries_bucket", + key=f"queries/{query_id}/data.json", + query=query, + ) + + except Exception as e: + logger.error(f"Unhandled status exception, error: {e}") + raise MoonstreamHTTPException(status_code=500) + + stats_presigned_url = s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": "queries_bucket", "Key": f"queries/{query_id}/data.json",}, + ExpiresIn=300000, + HttpMethod="GET", + ) + + return stats_presigned_url diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py b/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py new file mode 100644 index 00000000..94f09eb6 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py @@ -0,0 +1,35 @@ +import json +import logging +from typing import Any + +import boto3 # type: ignore +from moonstreamdb.db import yield_db_session_ctx + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def push_statistics(data: Any, key: str, bucket: str) -> None: + + result_bytes = json.dumps(data).encode("utf-8") + + s3 = boto3.client("s3") + s3.put_object( + Body=result_bytes, + Bucket=bucket, + Key=key, + ContentType="application/json", + Metadata={"drone_query": "data"}, + ) + + logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") + + +def data_generate(bucket: str, key: str, query: str): + """ + Generate query and push it to S3 + """ + with yield_db_session_ctx() as db_session: + push_statistics(data=db_session.execute(query).all(), key=key, bucket=bucket) + From e7a4fb684ee2f6bc5a713d30d4296f632fdb42a4 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:33:22 +0200 Subject: [PATCH 03/55] Add enb to settings. --- backend/moonstreamapi/settings.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 2882644a..2d646163 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -22,6 +22,12 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "") if MOONSTREAM_DATA_JOURNAL_ID == "": raise ValueError("MOONSTREAM_DATA_JOURNAL_ID environment variable must be set") + +MOONSTREAM_QUERIES_JOURNAL_ID = os.environ.get("MOONSTREAM_QUERIES_JOURNAL_ID", "") +if MOONSTREAM_DATA_JOURNAL_ID == "": + raise ValueError("MOONSTREAM_QUERIES_JOURNAL_ID environment variable must be set") + + MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") if MOONSTREAM_ADMIN_ACCESS_TOKEN == "": raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set") @@ -62,8 +68,8 @@ if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None: raise ValueError( "MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set" ) -MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = ( - MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/") +MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip( + "/" ) MOONSTREAM_CRAWLERS_SERVER_URL = os.environ.get("MOONSTREAM_CRAWLERS_SERVER_URL") From 8b26e386c83e9120837eb850555bd94875b46fb9 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:37:52 +0200 Subject: [PATCH 04/55] Add read only dubliction. --- db/moonstreamdb/db.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index bfe055e9..62e17fe6 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -11,6 +11,10 @@ MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") if MOONSTREAM_DB_URI is None: raise ValueError("MOONSTREAM_DB_URI environment variable must be set") +MOONSTREAM_DB_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_URI_READ_ONLY") +if MOONSTREAM_DB_URI_READ_ONLY is None: + raise ValueError("MOONSTREAM_DB_URI_READ_ONLY environment variable must be set") + MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0) try: if MOONSTREAM_POOL_SIZE_RAW is not None: @@ -60,3 +64,31 @@ def yield_db_session() -> Session: yield_db_session_ctx = contextmanager(yield_db_session) + + +# Read only + +RO_engine = create_engine( + MOONSTREAM_DB_URI_READ_ONLY, + pool_size=MOONSTREAM_POOL_SIZE, + connect_args={ + "options": f"-c statement_timeout={MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS}" + }, +) +RO_SessionLocal = sessionmaker(bind=RO_engine) + + +def yield_db_read_only_session() -> Session: + """ + Yields a database connection (created using environment variables). + As per FastAPI docs: + https://fastapi.tiangolo.com/tutorial/sql-databases/#create-a-dependency + """ + session = RO_SessionLocal() + try: + yield session + finally: + session.close() + + +yield_db_read_only_session_ctx = contextmanager(yield_db_session) From 3d1108e569c66e513aa66cc6750e85dabfeecf0e Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:39:49 +0200 Subject: [PATCH 05/55] Add json response from drones. --- crawlers/mooncrawl/mooncrawl/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 49e09436..6f13d915 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -192,4 +192,4 @@ async def queries_data_update_handler( HttpMethod="GET", ) - return stats_presigned_url + return {"url": stats_presigned_url} From 2b995c8d95715b7708813ddf03e9d94fa8bf8488 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:44:19 +0200 Subject: [PATCH 06/55] Add router. --- backend/moonstreamapi/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/moonstreamapi/api.py b/backend/moonstreamapi/api.py index 2f7db4f8..3caa8ecd 100644 --- a/backend/moonstreamapi/api.py +++ b/backend/moonstreamapi/api.py @@ -12,7 +12,8 @@ from . import actions, data from .middleware import BroodAuthMiddleware, MoonstreamHTTPException from .routes.address_info import router as addressinfo_router from .routes.dashboards import router as dashboards_router -from .routes.streams import router as streams_router +from .routes.queries import router as whales_router +from .routes.streams import router as queries_router from .routes.subscriptions import router as subscriptions_router from .routes.txinfo import router as txinfo_router from .routes.users import router as users_router @@ -30,6 +31,8 @@ tags_metadata = [ "name": "labels", "description": "Labels for transactions, addresses with additional information.", }, + {"name": "dashboards", "description": "Operations with user dashboards."}, + {"name": "queries", "description": "Operations with user queries."}, {"name": "streams", "description": "Operations with data streams and filters."}, {"name": "subscriptions", "description": "Operations with user subscriptions."}, {"name": "time", "description": "Server timestamp endpoints."}, @@ -127,3 +130,4 @@ app.include_router(txinfo_router) app.include_router(users_router) app.include_router(whales_router) app.include_router(dashboards_router) +app.include_router(queries_router) From 66d40335d62edd384a8fbfc7648b0c17e82095ab Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:46:24 +0200 Subject: [PATCH 07/55] Black formating. --- backend/moonstreamapi/routes/queries.py | 7 +++++-- backend/moonstreamapi/routes/streams.py | 8 ++++++-- backend/moonstreamapi/settings.py | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index edcdd724..a4300f8b 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -23,12 +23,15 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) @router.post("/{query_id}/update", tags=["queries"]) async def update_query_data_handler( - request: Request, query_id: str = Query(...), + request: Request, + query_id: str = Query(...), ) -> Optional[Dict[str, Any]]: """ Request update data on S3 bucket diff --git a/backend/moonstreamapi/routes/streams.py b/backend/moonstreamapi/routes/streams.py index 476a8b76..1917b6ea 100644 --- a/backend/moonstreamapi/routes/streams.py +++ b/backend/moonstreamapi/routes/streams.py @@ -30,7 +30,9 @@ from .subscriptions import BUGOUT_RESOURCE_TYPE_SUBSCRIPTION logger = logging.getLogger(__name__) -router = APIRouter(prefix="/streams",) +router = APIRouter( + prefix="/streams", +) def get_user_subscriptions(token: str) -> Dict[str, List[BugoutResource]]: @@ -39,7 +41,9 @@ def get_user_subscriptions(token: str) -> Dict[str, List[BugoutResource]]: """ response = bc.list_resources( token=token, - params={"type": BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,}, + params={ + "type": BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, + }, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 2d646163..4a3194c4 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -68,8 +68,8 @@ if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None: raise ValueError( "MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set" ) -MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip( - "/" +MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = ( + MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/") ) MOONSTREAM_CRAWLERS_SERVER_URL = os.environ.get("MOONSTREAM_CRAWLERS_SERVER_URL") From c41f8d1616cdd87c18cd6aaf0d934016bfe8234f Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:49:23 +0200 Subject: [PATCH 08/55] black formating crawlers. --- crawlers/mooncrawl/mooncrawl/api.py | 16 ++++++++++++---- .../mooncrawl/mooncrawl/stats_worker/quereis.py | 1 - 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 6f13d915..47561471 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -80,14 +80,17 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, + background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, + token=stats_update.token, + resource_id=stats_update.dashboard_id, + timeout=10, ) # get all user subscriptions @@ -167,7 +170,9 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, query: Any, background_tasks: BackgroundTasks, + query_id: str, + query: Any, + background_tasks: BackgroundTasks, ) -> str: s3_client = boto3.client("s3") @@ -187,7 +192,10 @@ async def queries_data_update_handler( stats_presigned_url = s3_client.generate_presigned_url( "get_object", - Params={"Bucket": "queries_bucket", "Key": f"queries/{query_id}/data.json",}, + Params={ + "Bucket": "queries_bucket", + "Key": f"queries/{query_id}/data.json", + }, ExpiresIn=300000, HttpMethod="GET", ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py b/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py index 94f09eb6..6ed9e6db 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py @@ -32,4 +32,3 @@ def data_generate(bucket: str, key: str, query: str): """ with yield_db_session_ctx() as db_session: push_statistics(data=db_session.execute(query).all(), key=key, bucket=bucket) - From 0cbd1774427e9453f4402ea776e33c2634c1f338 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:55:14 +0200 Subject: [PATCH 09/55] Add fixes. --- backend/moonstreamapi/api.py | 4 +-- backend/moonstreamapi/routes/queries.py | 13 ++++--- crawlers/mooncrawl/mooncrawl/api.py | 18 +++------- .../mooncrawl/stats_worker/quereis.py | 34 ------------------- 4 files changed, 13 insertions(+), 56 deletions(-) delete mode 100644 crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py diff --git a/backend/moonstreamapi/api.py b/backend/moonstreamapi/api.py index 3caa8ecd..0ba1fcee 100644 --- a/backend/moonstreamapi/api.py +++ b/backend/moonstreamapi/api.py @@ -12,8 +12,8 @@ from . import actions, data from .middleware import BroodAuthMiddleware, MoonstreamHTTPException from .routes.address_info import router as addressinfo_router from .routes.dashboards import router as dashboards_router -from .routes.queries import router as whales_router -from .routes.streams import router as queries_router +from .routes.queries import router as queries_router +from .routes.streams import router as streams_router from .routes.subscriptions import router as subscriptions_router from .routes.txinfo import router as txinfo_router from .routes.users import router as users_router diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index a4300f8b..ce3c462f 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -23,15 +23,12 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter( - prefix="/queries", -) +router = APIRouter(prefix="/queries",) @router.post("/{query_id}/update", tags=["queries"]) async def update_query_data_handler( - request: Request, - query_id: str = Query(...), + request: Request, query_id: str = Query(...), ) -> Optional[Dict[str, Any]]: """ Request update data on S3 bucket @@ -62,5 +59,7 @@ async def update_query_data_handler( ) return responce.json() - except: - return None + except Exception as e: + logger.error("Unable to get events") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + return None diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 47561471..54f32d9c 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -80,17 +80,14 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, - background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, - resource_id=stats_update.dashboard_id, - timeout=10, + token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, ) # get all user subscriptions @@ -170,10 +167,8 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, - query: Any, - background_tasks: BackgroundTasks, -) -> str: + query_id: str, query: Any, background_tasks: BackgroundTasks, +) -> Dict[str, Any]: s3_client = boto3.client("s3") @@ -192,10 +187,7 @@ async def queries_data_update_handler( stats_presigned_url = s3_client.generate_presigned_url( "get_object", - Params={ - "Bucket": "queries_bucket", - "Key": f"queries/{query_id}/data.json", - }, + Params={"Bucket": "queries_bucket", "Key": f"queries/{query_id}/data.json",}, ExpiresIn=300000, HttpMethod="GET", ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py b/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py deleted file mode 100644 index 6ed9e6db..00000000 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/quereis.py +++ /dev/null @@ -1,34 +0,0 @@ -import json -import logging -from typing import Any - -import boto3 # type: ignore -from moonstreamdb.db import yield_db_session_ctx - - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def push_statistics(data: Any, key: str, bucket: str) -> None: - - result_bytes = json.dumps(data).encode("utf-8") - - s3 = boto3.client("s3") - s3.put_object( - Body=result_bytes, - Bucket=bucket, - Key=key, - ContentType="application/json", - Metadata={"drone_query": "data"}, - ) - - logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") - - -def data_generate(bucket: str, key: str, query: str): - """ - Generate query and push it to S3 - """ - with yield_db_session_ctx() as db_session: - push_statistics(data=db_session.execute(query).all(), key=key, bucket=bucket) From 373f768a3f6aa9217cee70605c7cd9abf7a1c5db Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 02:57:39 +0200 Subject: [PATCH 10/55] Quries worker. --- .../mooncrawl/stats_worker/queries.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 crawlers/mooncrawl/mooncrawl/stats_worker/queries.py diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py new file mode 100644 index 00000000..6ed9e6db --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -0,0 +1,34 @@ +import json +import logging +from typing import Any + +import boto3 # type: ignore +from moonstreamdb.db import yield_db_session_ctx + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def push_statistics(data: Any, key: str, bucket: str) -> None: + + result_bytes = json.dumps(data).encode("utf-8") + + s3 = boto3.client("s3") + s3.put_object( + Body=result_bytes, + Bucket=bucket, + Key=key, + ContentType="application/json", + Metadata={"drone_query": "data"}, + ) + + logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") + + +def data_generate(bucket: str, key: str, query: str): + """ + Generate query and push it to S3 + """ + with yield_db_session_ctx() as db_session: + push_statistics(data=db_session.execute(query).all(), key=key, bucket=bucket) From 5485a73b98577e9fbfaebbdd07bf5b4227ca3a17 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 03:00:49 +0200 Subject: [PATCH 11/55] Add black formating. --- backend/moonstreamapi/routes/queries.py | 7 +++++-- crawlers/mooncrawl/mooncrawl/api.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index ce3c462f..f06fee1e 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -23,12 +23,15 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) @router.post("/{query_id}/update", tags=["queries"]) async def update_query_data_handler( - request: Request, query_id: str = Query(...), + request: Request, + query_id: str = Query(...), ) -> Optional[Dict[str, Any]]: """ Request update data on S3 bucket diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 54f32d9c..713a6eba 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -80,14 +80,17 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, + background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, + token=stats_update.token, + resource_id=stats_update.dashboard_id, + timeout=10, ) # get all user subscriptions @@ -167,7 +170,9 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, query: Any, background_tasks: BackgroundTasks, + query_id: str, + query: Any, + background_tasks: BackgroundTasks, ) -> Dict[str, Any]: s3_client = boto3.client("s3") @@ -187,7 +192,10 @@ async def queries_data_update_handler( stats_presigned_url = s3_client.generate_presigned_url( "get_object", - Params={"Bucket": "queries_bucket", "Key": f"queries/{query_id}/data.json",}, + Params={ + "Bucket": "queries_bucket", + "Key": f"queries/{query_id}/data.json", + }, ExpiresIn=300000, HttpMethod="GET", ) From 025baeb1c4503091890fe9452ec9239c1beea11d Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 18:12:42 +0200 Subject: [PATCH 12/55] Add test workable version. --- backend/configs/sample.env | 2 + backend/moonstreamapi/data.py | 4 ++ backend/moonstreamapi/routes/queries.py | 42 ++++++++------- crawlers/mooncrawl/mooncrawl/api.py | 24 ++++----- crawlers/mooncrawl/mooncrawl/data.py | 11 +++- crawlers/mooncrawl/mooncrawl/settings.py | 6 +++ .../mooncrawl/stats_worker/queries.py | 53 +++++++++++++++---- crawlers/mooncrawl/sample.env | 1 + 8 files changed, 101 insertions(+), 42 deletions(-) diff --git a/backend/configs/sample.env b/backend/configs/sample.env index 07bcb5d6..8a6b4cdf 100644 --- a/backend/configs/sample.env +++ b/backend/configs/sample.env @@ -12,6 +12,8 @@ export MOONSTREAM_MOONWORM_TASKS_JOURNAL=" Optional[Dict[str, Any]]: """ Request update data on S3 bucket """ - token = request.state.token - try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved #query:{query_id} #user_token:{token}", + query=f"#approved #query:{query_id}", limit=1, + timeout=5, ) if entries.results and entries.results[0].content: content = entries.results[0].content + tags = entries.results[0].tags + + file_type = "json" + + if "ext:csv" in tags: + file_type = "csv" + responce = requests.post( - f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/query_update", - json=content, + f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/{query_id}/query_update", + json={ + "query": content, + "params": request_update.params, + "file_type": file_type, + }, + timeout=5, ) if responce.status_code != 200: @@ -63,6 +69,6 @@ async def update_query_data_handler( return responce.json() except Exception as e: - logger.error("Unable to get events") + logger.error(f"Error in send generate query data task: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) return None diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 713a6eba..abdd8cd0 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -21,6 +21,7 @@ from .settings import ( ORIGINS, bugout_client as bc, BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, + MOONSTREAM_QUERIES_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from .version import MOONCRAWL_VERSION @@ -80,17 +81,14 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, - background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, - resource_id=stats_update.dashboard_id, - timeout=10, + token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, ) # get all user subscriptions @@ -170,9 +168,7 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, - query: Any, - background_tasks: BackgroundTasks, + query_id: str, request: data.QueryDataUpdate, background_tasks: BackgroundTasks, ) -> Dict[str, Any]: s3_client = boto3.client("s3") @@ -181,9 +177,11 @@ async def queries_data_update_handler( background_tasks.add_task( queries.data_generate, - bucket="queries_bucket", - key=f"queries/{query_id}/data.json", - query=query, + bucket=MOONSTREAM_QUERIES_BUCKET, + query_id=f"{query_id}", + file_type=request.file_type, + query=request.query, + params=request.params, ) except Exception as e: @@ -193,8 +191,8 @@ async def queries_data_update_handler( stats_presigned_url = s3_client.generate_presigned_url( "get_object", Params={ - "Bucket": "queries_bucket", - "Key": f"queries/{query_id}/data.json", + "Bucket": MOONSTREAM_QUERIES_BUCKET, + "Key": f"queries/{query_id}/data.{request.file_type}", }, ExpiresIn=300000, HttpMethod="GET", diff --git a/crawlers/mooncrawl/mooncrawl/data.py b/crawlers/mooncrawl/mooncrawl/data.py index 4bf694c9..ec0b1859 100644 --- a/crawlers/mooncrawl/mooncrawl/data.py +++ b/crawlers/mooncrawl/mooncrawl/data.py @@ -1,9 +1,9 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import List +from typing import List, Any, Dict -from pydantic import BaseModel +from pydantic import BaseModel, Field class AvailableBlockchainType(Enum): @@ -47,3 +47,10 @@ class NowResponse(BaseModel): """ epoch_time: float + + +class QueryDataUpdate(BaseModel): + + file_type: str + query: str + params: Dict[str, Any] = Field(default_factory=dict) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 89d2751f..f85575a2 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -84,3 +84,9 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "": raise ValueError( "MOONSTREAM_MOONWORM_TASKS_JOURNAL environment variable must be set" ) + + +MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") +if MOONSTREAM_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") + diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 6ed9e6db..782bd293 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -1,8 +1,11 @@ import json import logging -from typing import Any +from typing import Any, Dict, Optional +from io import StringIO +import csv -import boto3 # type: ignore + +import boto3 from moonstreamdb.db import yield_db_session_ctx @@ -10,13 +13,10 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def push_statistics(data: Any, key: str, bucket: str) -> None: +def push_statistics(s3: Any, data: Any, key: str, bucket: str) -> None: - result_bytes = json.dumps(data).encode("utf-8") - - s3 = boto3.client("s3") s3.put_object( - Body=result_bytes, + Body=data, Bucket=bucket, Key=key, ContentType="application/json", @@ -26,9 +26,44 @@ def push_statistics(data: Any, key: str, bucket: str) -> None: logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") -def data_generate(bucket: str, key: str, query: str): +def data_generate( + bucket: str, + query_id: str, + file_type: str, + query: str, + params: Optional[Dict[str, Any]], +): """ Generate query and push it to S3 """ + s3 = boto3.client("s3") + with yield_db_session_ctx() as db_session: - push_statistics(data=db_session.execute(query).all(), key=key, bucket=bucket) + + if file_type == "csv": + csv_buffer = StringIO() + csv_writer = csv.writer(csv_buffer, delimiter=";") + + # engine.execution_options(stream_results=True) + result = db_session.execute(query, params).keys() + + csv_writer.writerow(result.keys()) + csv_writer.writerows(result.fetchAll()) + + push_statistics( + s3=s3, + data=csv_buffer.getvalue().encode("utf-8"), + key=f"queries/{query_id}/data.{file_type}", + bucket=bucket, + ) + else: + + data = json.dumps( + [dict(row) for row in db_session.execute(query, params)] + ).encode("utf-8") + push_statistics( + s3=s3, + data=data, + key=f"queries/{query_id}/data.{file_type}", + bucket=bucket, + ) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a22bb6b7..73e388f8 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -15,3 +15,4 @@ export MOONSTREAM_DATA_JOURNAL_ID="" export MOONSTREAM_ADMIN_ACCESS_TOKEN="" export NFT_HUMBUG_TOKEN="" export MOONSTREAM_MOONWORM_TASKS_JOURNAL="" +export MOONSTREAM_QUERIES_BUCKET="" From 4dd101eb436765bf7a049ce21573709cbd7eb463 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Wed, 16 Feb 2022 18:23:29 +0200 Subject: [PATCH 13/55] Add lint fixes. --- backend/moonstreamapi/routes/queries.py | 4 +++- crawlers/mooncrawl/mooncrawl/api.py | 11 ++++++++--- crawlers/mooncrawl/mooncrawl/settings.py | 1 - crawlers/mooncrawl/mooncrawl/stats_worker/queries.py | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 39e8a491..f5e57d75 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -21,7 +21,9 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) @router.post("/{query_id}/update", tags=["queries"]) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index abdd8cd0..a52e973c 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -81,14 +81,17 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, + background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, + token=stats_update.token, + resource_id=stats_update.dashboard_id, + timeout=10, ) # get all user subscriptions @@ -168,7 +171,9 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, request: data.QueryDataUpdate, background_tasks: BackgroundTasks, + query_id: str, + request: data.QueryDataUpdate, + background_tasks: BackgroundTasks, ) -> Dict[str, Any]: s3_client = boto3.client("s3") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index f85575a2..66b01bdc 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -89,4 +89,3 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "": MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") if MOONSTREAM_QUERIES_BUCKET == "": raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") - diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 782bd293..1b6d89a6 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -5,7 +5,7 @@ from io import StringIO import csv -import boto3 +import boto3 # type: ignore from moonstreamdb.db import yield_db_session_ctx From 1c741d50844e822d55c1af9439d9d0e914f2925c Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 12:22:20 +0200 Subject: [PATCH 14/55] Add create update get endpoints. --- backend/moonstreamapi/actions.py | 33 +++- backend/moonstreamapi/data.py | 13 +- backend/moonstreamapi/routes/queries.py | 232 +++++++++++++++++++++++- backend/moonstreamapi/settings.py | 6 + crawlers/mooncrawl/mooncrawl/api.py | 10 +- 5 files changed, 283 insertions(+), 11 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 8468b00b..7bc5d606 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -8,8 +8,14 @@ import uuid import boto3 # type: ignore -from bugout.data import BugoutSearchResults, BugoutSearchResult, BugoutResource +from bugout.data import ( + BugoutSearchResults, + BugoutSearchResult, + BugoutResource, + BugoutResources, +) from bugout.journal import SearchOrder +from bugout.exceptions import BugoutResponseException from ens.utils import is_valid_ens_name # type: ignore from eth_utils.address import is_address # type: ignore from moonstreamdb.models import EthereumLabel @@ -30,6 +36,7 @@ from .settings import ( MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, MOONSTREAM_MOONWORM_TASKS_JOURNAL, + BUGOUT_RESOURCE_QUERY_RESOLVER, ) from .settings import bugout_client as bc @@ -535,3 +542,27 @@ def apply_moonworm_tasks( entries=entries_pack, timeout=15, ) + + +def get_query_by_name(query_name: str, token: uuid.UUID) -> str: + + params = {"type": BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error get query, error: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + available_queries: Dict[str, str] = { + resource.resource_data["name"]: resource.resource_data["entry_id"] + for resource in resources.resources + } + + if query_name not in available_queries: + raise MoonstreamHTTPException(status_code=404, detail="Query not found.") + + query_id = available_queries[query_name] + + return query_id diff --git a/backend/moonstreamapi/data.py b/backend/moonstreamapi/data.py index 61d0b0fd..346504db 100644 --- a/backend/moonstreamapi/data.py +++ b/backend/moonstreamapi/data.py @@ -3,10 +3,12 @@ Pydantic schemas for the Moonstream HTTP API """ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Literal from uuid import UUID +from xmlrpc.client import Boolean from pydantic import BaseModel, Field +from sqlalchemy import false USER_ONBOARDING_STATE = "onboarding_state" @@ -266,3 +268,12 @@ class DashboardUpdate(BaseModel): class UpdateDataRequest(BaseModel): params: Dict[str, Any] = Field(default_factory=dict) + + +class UpdateQueryRequest(BaseModel): + query: str + + +class PreapprovedQuery(BaseModel): + query: str + public: bool = False diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index f5e57d75..18763484 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -2,19 +2,25 @@ The Moonstream queries HTTP API """ import logging -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, Body +import boto3 # type: ignore +from bugout.data import BugoutResources +from bugout.exceptions import BugoutResponseException +from fastapi import APIRouter, Body, Request import requests from .. import data +from ..actions import get_query_by_name from ..middleware import MoonstreamHTTPException from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_QUERIES_JOURNAL_ID, + MOONSTREAM_APPLICATION_ID, MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_CRAWLERS_SERVER_PORT, + MOONSTREAM_QUERIES_BUCKET, + MOONSTREAM_QUERIES_JOURNAL_ID, + BUGOUT_RESOURCE_QUERY_RESOLVER, ) from ..settings import bugout_client as bc @@ -26,19 +32,178 @@ router = APIRouter( ) -@router.post("/{query_id}/update", tags=["queries"]) +BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" + + +@router.post("/{query_name}", tags=["queries"]) +async def create_query_handler( + request: Request, query_name: str, query_applied: data.PreapprovedQuery = Body(...) +) -> Any: + """ + Create query in bugout journal + """ + + token = request.state.token + + user = request.state.user + + # Check already existed queries + + params = { + "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + } + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error( + f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" + ) + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + used_queries: List[str] = [ + resource.resource_data["name"] for resource in resources.resources + ] + + if query_name in used_queries: + + raise MoonstreamHTTPException( + status_code=404, + detail=f"Provided query name already use. Please remove it or use PUT /{query_name}", + ) + + try: + # Put query to journal + entry = bc.create_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + title=f"Query:{query_name}", + tags=["type:query"], + content=query_applied.query, + ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error creating query entry: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + # create resource query_name_resolver + bc.create_resource( + token=token, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data={ + "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + "user_id": str(user.id), + "name": query_name, + "entry_id": str(entry.id), + }, + ) + except BugoutResponseException as e: + logger.error(f"Error creating name resolving resource: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error creating name resolving resource: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + + bc.update_tags( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=entry.id, + tags=[f"query_id:{entry.id}", f"preapprove"], + ) + + except BugoutResponseException as e: + logger.error(f"Error in applind tags to query entry: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error in applind tags to query entry: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return True + + +@router.get("/{query_name}/query", tags=["queries"]) +async def get_query_handler(request: Request, query_name: str) -> Any: + + token = request.state.token + + query_id = get_query_by_name(query_name, token) + + try: + + entry = bc.get_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_id, + ) + + except BugoutResponseException as e: + logger.error(f"Error in updating query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + + except Exception as e: + logger.error(f"Error in updating query: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + + +@router.put("/{query_name}", tags=["queries"]) +async def update_query_handler( + request: Request, + query_name: str, + request_update: data.UpdateQueryRequest = Body(...), +) -> Any: + + token = request.state.token + + query_id = get_query_by_name(query_name, token) + + try: + + entry = bc.update_entry_content( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_id, + title=query_name, + content=request_update.query, + tags=["preapprove"], + ) + + except BugoutResponseException as e: + logger.error(f"Error in updating query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + + except Exception as e: + logger.error(f"Error in updating query: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + + +@router.post("/{query_name}/update_data", tags=["queries"]) async def update_query_data_handler( - query_id: str, request_update: data.UpdateDataRequest = Body(...) + request: Request, + query_name: str, + request_update: data.UpdateDataRequest = Body(...), ) -> Optional[Dict[str, Any]]: """ Request update data on S3 bucket """ + token = request.state.token + + query_id = get_query_by_name(query_name, token) + try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved #query:{query_id}", + query=f"#approved ! #query_id:{query_id}", limit=1, timeout=5, ) @@ -73,4 +238,55 @@ async def update_query_data_handler( except Exception as e: logger.error(f"Error in send generate query data task: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) - return None + raise MoonstreamHTTPException(status_code=403, detail="Query not approved yet.") + + +@router.get("/{query_name}", tags=["queries"]) +async def get_access_link_handler( + request: Request, + query_name: str, +) -> str: + """ + Request update data on S3 bucket + """ + + # get real connect to query_id + + token = request.state.token + + query_id = get_query_by_name(query_name, token) + + s3 = boto3.client("s3") + + try: + entries = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + query=f"#approved #query_id:{query_id}", + limit=1, + timeout=5, + ) + + if entries.results and entries.results[0].content: + + tags = entries.results[0].tags + + file_type = "json" + + if "ext:csv" in tags: + file_type = "csv" + + stats_presigned_url = s3.generate_presigned_url( + "get_object", + Params={ + "Bucket": MOONSTREAM_QUERIES_BUCKET, + "Key": f"queries/{query_id}/data.{file_type}", + }, + ExpiresIn=300000, + HttpMethod="GET", + ) + return stats_presigned_url + except Exception as e: + logger.error(f"Error in send generate query data task: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + raise MoonstreamHTTPException(status_code=403, detail="Query not approved yet.") diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 4a3194c4..88935592 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -7,6 +7,8 @@ BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") +BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" + bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 @@ -99,3 +101,7 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": raise ValueError( "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable must be set" ) + +MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") +if MOONSTREAM_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index a52e973c..38d97535 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -11,6 +11,7 @@ from uuid import UUID import boto3 # type: ignore from fastapi import FastAPI, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy import text from bugout.data import BugoutResource, BugoutResources @@ -180,6 +181,13 @@ async def queries_data_update_handler( try: + # test statement params + t = text(request.query) + t = t.bindparams(**request.params) + + t.compile(compile_kwargs={"literal_binds": True}) + print(t.compile(compile_kwargs={"literal_binds": True})) + background_tasks.add_task( queries.data_generate, bucket=MOONSTREAM_QUERIES_BUCKET, @@ -199,7 +207,7 @@ async def queries_data_update_handler( "Bucket": MOONSTREAM_QUERIES_BUCKET, "Key": f"queries/{query_id}/data.{request.file_type}", }, - ExpiresIn=300000, + ExpiresIn=43200, # 12 hours HttpMethod="GET", ) From 1498d1bac6edd01a6d0556f04029263a59e3824a Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 12:24:14 +0200 Subject: [PATCH 15/55] remove testing. --- crawlers/mooncrawl/mooncrawl/api.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 38d97535..f511259c 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -181,13 +181,6 @@ async def queries_data_update_handler( try: - # test statement params - t = text(request.query) - t = t.bindparams(**request.params) - - t.compile(compile_kwargs={"literal_binds": True}) - print(t.compile(compile_kwargs={"literal_binds": True})) - background_tasks.add_task( queries.data_generate, bucket=MOONSTREAM_QUERIES_BUCKET, From 86adb30f2ae3cf848bb828c7953153f9321d825c Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 12:32:23 +0200 Subject: [PATCH 16/55] Add anotation fix. --- backend/moonstreamapi/routes/queries.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 18763484..8fc6aace 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -5,7 +5,7 @@ import logging from typing import Any, Dict, List, Optional import boto3 # type: ignore -from bugout.data import BugoutResources +from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Request import requests @@ -38,7 +38,7 @@ BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @router.post("/{query_name}", tags=["queries"]) async def create_query_handler( request: Request, query_name: str, query_applied: data.PreapprovedQuery = Body(...) -) -> Any: +) -> BugoutJournalEntry: """ Create query in bugout journal """ @@ -123,11 +123,11 @@ async def create_query_handler( logger.error(f"Error in applind tags to query entry: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) - return True + return entry @router.get("/{query_name}/query", tags=["queries"]) -async def get_query_handler(request: Request, query_name: str) -> Any: +async def get_query_handler(request: Request, query_name: str) -> BugoutJournalEntry: token = request.state.token @@ -157,7 +157,7 @@ async def update_query_handler( request: Request, query_name: str, request_update: data.UpdateQueryRequest = Body(...), -) -> Any: +) -> BugoutJournalEntryContent: token = request.state.token From 1fe78d33564eef93f1c31696d3868513325fa7fe Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 15:10:32 +0200 Subject: [PATCH 17/55] Add list endpoint for accessable of user queries. --- backend/moonstreamapi/routes/queries.py | 36 ++++++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 8fc6aace..9323e2ad 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -27,14 +27,38 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter( - prefix="/queries", -) +router = APIRouter(prefix="/queries",) BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" +@router.get("/list", tags=["queries"]) +async def get_access_link_handler(request: Request) -> Dict[str, Any]: + + token = request.state.token + + # Check already existed queries + + params = { + "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + } + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error( + f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" + ) + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + users_queries: Dict[str, Any] = [ + resource.resource_data for resource in resources.resources + ] + return users_queries + + @router.post("/{query_name}", tags=["queries"]) async def create_query_handler( request: Request, query_name: str, query_applied: data.PreapprovedQuery = Body(...) @@ -96,6 +120,7 @@ async def create_query_handler( resource_data={ "type": BUGOUT_RESOURCE_QUERY_RESOLVER, "user_id": str(user.id), + "user": str(user.username), "name": query_name, "entry_id": str(entry.id), }, @@ -242,10 +267,7 @@ async def update_query_data_handler( @router.get("/{query_name}", tags=["queries"]) -async def get_access_link_handler( - request: Request, - query_name: str, -) -> str: +async def get_access_link_handler(request: Request, query_name: str,) -> str: """ Request update data on S3 bucket """ From 82f7882a511b71a2fadf6932cfc2f77c7308c784 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 15:24:11 +0200 Subject: [PATCH 18/55] Add mypy fix. --- backend/moonstreamapi/routes/queries.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 9323e2ad..2b6da906 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -27,14 +27,16 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @router.get("/list", tags=["queries"]) -async def get_access_link_handler(request: Request) -> Dict[str, Any]: +async def get_list_of_queries_handler(request: Request) -> Dict[str, Any]: token = request.state.token @@ -53,7 +55,7 @@ async def get_access_link_handler(request: Request) -> Dict[str, Any]: ) raise MoonstreamHTTPException(status_code=500, internal_error=e) - users_queries: Dict[str, Any] = [ + users_queries: List[Dict[str, Any]] = [ resource.resource_data for resource in resources.resources ] return users_queries @@ -267,7 +269,10 @@ async def update_query_data_handler( @router.get("/{query_name}", tags=["queries"]) -async def get_access_link_handler(request: Request, query_name: str,) -> str: +async def get_access_link_handler( + request: Request, + query_name: str, +) -> str: """ Request update data on S3 bucket """ From d031a710136dec2a3f384fe27b7060aa7233018d Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 17 Feb 2022 17:26:48 +0200 Subject: [PATCH 19/55] Add temp solution for show block and timestamp. --- crawlers/mooncrawl/mooncrawl/stats_worker/queries.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 1b6d89a6..57f87aea 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -7,6 +7,7 @@ import csv import boto3 # type: ignore from moonstreamdb.db import yield_db_session_ctx +from sqlalchemy import text logging.basicConfig(level=logging.INFO) @@ -57,9 +58,16 @@ def data_generate( bucket=bucket, ) else: + block_number, block_timestamp = db_session.execute( + "SELECT block_number, block_timestamp FROM polygon_labels WHERE block_number=(SELECT max(block_number) FROM polygon_labels where label='moonworm-alpha') limit 1;", + ).one() data = json.dumps( - [dict(row) for row in db_session.execute(query, params)] + { + "block_number": block_number, + "block_timestamp": block_timestamp, + "data": [dict(row) for row in db_session.execute(query, params)], + } ).encode("utf-8") push_statistics( s3=s3, From c5211898405280506ed8f55044f3d160698880e6 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Fri, 18 Feb 2022 19:48:49 +0200 Subject: [PATCH 20/55] Add slugify Lib for normalize query name. Also add remove query endpoint. --- backend/moonstreamapi/data.py | 1 + backend/moonstreamapi/routes/queries.py | 76 ++++++++++++++++++++++--- backend/setup.py | 1 + 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/backend/moonstreamapi/data.py b/backend/moonstreamapi/data.py index 346504db..5c497330 100644 --- a/backend/moonstreamapi/data.py +++ b/backend/moonstreamapi/data.py @@ -276,4 +276,5 @@ class UpdateQueryRequest(BaseModel): class PreapprovedQuery(BaseModel): query: str + name: str public: bool = False diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 2b6da906..0a70eb4d 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -2,13 +2,16 @@ The Moonstream queries HTTP API """ import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple, Union +from uuid import UUID import boto3 # type: ignore from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Request import requests +from slugify import slugify + from .. import data from ..actions import get_query_by_name @@ -27,9 +30,7 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter( - prefix="/queries", -) +router = APIRouter(prefix="/queries",) BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @@ -61,7 +62,7 @@ async def get_list_of_queries_handler(request: Request) -> Dict[str, Any]: return users_queries -@router.post("/{query_name}", tags=["queries"]) +@router.post("/", tags=["queries"]) async def create_query_handler( request: Request, query_name: str, query_applied: data.PreapprovedQuery = Body(...) ) -> BugoutJournalEntry: @@ -92,6 +93,8 @@ async def create_query_handler( resource.resource_data["name"] for resource in resources.resources ] + query_name = slugify(query_applied.name) + if query_name in used_queries: raise MoonstreamHTTPException( @@ -269,10 +272,7 @@ async def update_query_data_handler( @router.get("/{query_name}", tags=["queries"]) -async def get_access_link_handler( - request: Request, - query_name: str, -) -> str: +async def get_access_link_handler(request: Request, query_name: str,) -> str: """ Request update data on S3 bucket """ @@ -317,3 +317,61 @@ async def get_access_link_handler( logger.error(f"Error in send generate query data task: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) raise MoonstreamHTTPException(status_code=403, detail="Query not approved yet.") + + +@router.delete("/{query_name}", tags=["queries"]) +async def remove_query_handler( + request: Request, query_name: str, +) -> BugoutJournalEntry: + """ + Request update data on S3 bucket + """ + token = request.state.token + + """ + def delete_resource( + self, + token: Union[str, uuid.UUID], + resource_id: Union[str, uuid.UUID], + timeout: float = REQUESTS_TIMEOUT + """ + + params = {"type": BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error get query, error: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + query_ids: Dict[str, Tuple[UUID, Union[UUID, str]]] = { + resource.resource_data["name"]: ( + resource.id, + resource.resource_data["entry_id"], + ) + for resource in resources.resources + } + + try: + bc.remove_resources(token=token, resource_id=query_ids[query_name][0]) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error get query, error: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + entry = bc.delete_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_ids[query_name][1], + ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error get query, error: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + diff --git a/backend/setup.py b/backend/setup.py index 705a2b3c..70b3ed44 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -21,6 +21,7 @@ setup( "pyevmasm", "python-dateutil", "python-multipart", + "python-slugify", "uvicorn", "web3", ], From a1595e3e269be4d34d134f8922f39dbd412c46b2 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Fri, 18 Feb 2022 19:51:33 +0200 Subject: [PATCH 21/55] Add not exist case for delete endpoint. --- backend/moonstreamapi/routes/queries.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 0a70eb4d..397dc2b9 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -352,6 +352,8 @@ async def remove_query_handler( ) for resource in resources.resources } + if len(query_ids) == 0: + raise MoonstreamHTTPException(status_code=404, detail="Query does not existю") try: bc.remove_resources(token=token, resource_id=query_ids[query_name][0]) From ef04be14f507f5f0a8583290120daed5fdb23eb8 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Fri, 18 Feb 2022 19:58:16 +0200 Subject: [PATCH 22/55] Add mypy fixes. --- backend/moonstreamapi/routes/queries.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 397dc2b9..67936b37 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -10,7 +10,7 @@ from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJourna from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Request import requests -from slugify import slugify +from slugify import slugify # type: ignore from .. import data @@ -30,14 +30,16 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @router.get("/list", tags=["queries"]) -async def get_list_of_queries_handler(request: Request) -> Dict[str, Any]: +async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: token = request.state.token @@ -233,7 +235,7 @@ async def update_query_data_handler( entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved ! #query_id:{query_id}", + query=f"#approved #query_id:{query_id} !#preapprove", limit=1, timeout=5, ) @@ -272,7 +274,10 @@ async def update_query_data_handler( @router.get("/{query_name}", tags=["queries"]) -async def get_access_link_handler(request: Request, query_name: str,) -> str: +async def get_access_link_handler( + request: Request, + query_name: str, +) -> str: """ Request update data on S3 bucket """ @@ -289,7 +294,7 @@ async def get_access_link_handler(request: Request, query_name: str,) -> str: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved #query_id:{query_id}", + query=f"#approved #query_id:{query_id} !#preapprove", limit=1, timeout=5, ) @@ -321,7 +326,8 @@ async def get_access_link_handler(request: Request, query_name: str,) -> str: @router.delete("/{query_name}", tags=["queries"]) async def remove_query_handler( - request: Request, query_name: str, + request: Request, + query_name: str, ) -> BugoutJournalEntry: """ Request update data on S3 bucket @@ -356,7 +362,7 @@ async def remove_query_handler( raise MoonstreamHTTPException(status_code=404, detail="Query does not existю") try: - bc.remove_resources(token=token, resource_id=query_ids[query_name][0]) + bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: @@ -376,4 +382,3 @@ async def remove_query_handler( raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry - From 45cc5285827aeef59d8edf0c749432221c29367c Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Mar 2022 20:11:40 +0000 Subject: [PATCH 23/55] ldb blockchain database crawler --- crawlers/ldb/cmd/chain.go | 217 +++++++++++ crawlers/ldb/cmd/cli.go | 176 +++++++++ crawlers/ldb/configs/settings.go | 11 + crawlers/ldb/go.mod | 63 +++ crawlers/ldb/go.sum | 633 +++++++++++++++++++++++++++++++ crawlers/ldb/main.go | 9 + crawlers/ldb/sample.env | 1 + 7 files changed, 1110 insertions(+) create mode 100644 crawlers/ldb/cmd/chain.go create mode 100644 crawlers/ldb/cmd/cli.go create mode 100644 crawlers/ldb/configs/settings.go create mode 100644 crawlers/ldb/go.mod create mode 100644 crawlers/ldb/go.sum create mode 100644 crawlers/ldb/main.go create mode 100644 crawlers/ldb/sample.env diff --git a/crawlers/ldb/cmd/chain.go b/crawlers/ldb/cmd/chain.go new file mode 100644 index 00000000..f530be4a --- /dev/null +++ b/crawlers/ldb/cmd/chain.go @@ -0,0 +1,217 @@ +package cmd + +import ( + "database/sql" + "fmt" + "time" + + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + _ "github.com/lib/pq" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/node" + "gopkg.in/urfave/cli.v1" +) + +type gethConfig struct { + Eth ethconfig.Config + Node node.Config +} + +func defaultNodeConfig() node.Config { + cfg := node.DefaultConfig + cfg.Name = "geth" + return cfg +} + +type LocalConnections struct { + Stack *node.Node + Chain *core.BlockChain + ChainDB ethdb.Database + Database *sql.DB +} + +func setLocalChain(ctx *cli.Context) error { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + } + + // Apply flags + utils.SetNodeConfig(ctx, &cfg.Node) + stack, err := node.New(&cfg.Node) + if err != nil { + return fmt.Errorf("Failed to create the protocol stack: %v", err) + } + localConnections.Stack = stack + + utils.SetEthConfig(ctx, stack, &cfg.Eth) + + chain, chainDB := utils.MakeChain(ctx, stack) + localConnections.Chain = chain + localConnections.ChainDB = chainDB + + return nil +} + +func setDatabase() error { + db, err := sql.Open("postgres", configs.MOONSTREAM_DB_URI) + if err != nil { + return fmt.Errorf("DSN parse error or another database initialization error: %v", err) + } + + // Set the maximum number of concurrently idle connections, + // by default sql.DB allows a maximum of 2 idle connections. + db.SetMaxIdleConns(configs.MOONSTREAM_DB_MAX_IDLE_CONNS) + + // Set the maximum lifetime of a connection. + // Longer lifetime increase memory usage. + db.SetConnMaxLifetime(configs.MOONSTREAM_DB_CONN_MAX_LIFETIME) + + localConnections.Database = db + + return nil +} + +func (lc *LocalConnections) fetchFromNode(start, end uint64) error { + for i := start; i <= end; i++ { + header := lc.Chain.GetHeaderByNumber(i) + blockHeaderHash := header.Hash() + body := rawdb.ReadBody(lc.ChainDB, blockHeaderHash, i) + var txs []common.Hash + for _, tx := range body.Transactions { + txs = append(txs, tx.Hash()) + } + + fmt.Printf("- Block header with hash: %s and transactions: %s\n", blockHeaderHash, txs) + } + + return nil +} + +type Block struct { + Hash string `json:"hash"` + BlockNumber uint64 `json:"block_number"` +} + +type Transactions struct { + Txs []string + Quantity int +} + +// Retrive block from blockchain +func (lc *LocalConnections) getBlockFromChain(number uint64) (*types.Header, error) { + header := lc.Chain.GetHeaderByNumber(number) + if header == nil { + return nil, fmt.Errorf("Not found %d block in chain", number) + } + return header, nil +} + +// Retrieve block from database +func (lc *LocalConnections) getBlockFromDB(header *types.Header) (Block, error) { + var block Block + blockRow := lc.Database.QueryRow(fmt.Sprintf("SELECT hash,block_number FROM ethereum_blocks WHERE hash = '%s';", header.Hash().String())) + err := blockRow.Scan(&block.Hash, &block.BlockNumber) + if err != nil { + if err == sql.ErrNoRows { + return block, fmt.Errorf("Not found %d block: %v", header.Number, err) + } + return block, fmt.Errorf("An error occurred during sql operation: %v", err) + } + + return block, nil +} + +// Retrive block transactions from blockchain +func (lc *LocalConnections) getTxsFromChain(headerHash common.Hash, number uint64) Transactions { + var transactions Transactions + body := rawdb.ReadBody(lc.ChainDB, headerHash, number) + for _, tx := range body.Transactions { + transactions.Txs = append(transactions.Txs, tx.Hash().String()) + transactions.Quantity++ + // set[tx.Hash().String()] = false + } + + return transactions +} + +// Retrive block transactions from database +func (lc *LocalConnections) getTxsFromDB(blockNumber uint64) (Transactions, error) { + var transactions Transactions + rows, err := lc.Database.Query(fmt.Sprintf("SELECT hash FROM ethereum_transactions WHERE block_number = %d;", blockNumber)) + if err != nil { + return transactions, fmt.Errorf("An error occurred during sql operation: %v", err) + } + defer rows.Close() + + for rows.Next() { + var txHash string + err := rows.Scan(&txHash) + if err != nil { + return transactions, fmt.Errorf("An error occurred during sql operation: %v", err) + } + transactions.Txs = append(transactions.Txs, txHash) + transactions.Quantity++ + // set[transaction] = true + } + return transactions, nil +} + +// Write down inconsistent state between database and blockchain +// **source** (string): Source of nonconformity [blockchain, database] +func recordNonconformity(number uint64, source string) { + fmt.Println(number, source) +} + +func verify(start, end uint64) error { + for i := start; i <= end; i++ { + header, err := localConnections.getBlockFromChain(i) + if err != nil { + fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) + recordNonconformity(i, "blockchain") + continue + } + + block, err := localConnections.getBlockFromDB(header) + if err != nil { + fmt.Printf("Unable to get block: %d, err: %v\n", block.BlockNumber, err) + recordNonconformity(block.BlockNumber, "database") + continue + } + + if header.Number.Uint64() != block.BlockNumber { + fmt.Printf("Incorrect %d block retrieved from database\n", block.BlockNumber) + recordNonconformity(block.BlockNumber, "database") + continue + } + + // set := make(map[string]bool) + chainTxs := localConnections.getTxsFromChain(header.Hash(), i) + + dbTxs, err := localConnections.getTxsFromDB(header.Number.Uint64()) + if err != nil { + fmt.Printf("Unable to get transactions: %d, err: %v\n", block.BlockNumber, err) + recordNonconformity(block.BlockNumber, "database") + continue + } + + if chainTxs.Quantity != dbTxs.Quantity { + fmt.Printf("Different number of transactions in block %d, err %v\n", block.BlockNumber, err) + recordNonconformity(block.BlockNumber, "database") + continue + } + + fmt.Printf("Processed block number: %d\r", i) + time.Sleep(1 * time.Second) + } + fmt.Println("") + + return nil +} diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go new file mode 100644 index 00000000..02445ee0 --- /dev/null +++ b/crawlers/ldb/cmd/cli.go @@ -0,0 +1,176 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strconv" + + "github.com/ethereum/go-ethereum/cmd/utils" + "gopkg.in/urfave/cli.v1" +) + +var ( + DataDirFlag = cli.StringFlag{ + Name: "datadir", + Usage: "Data directory for the databases and keystore", + Value: "/home/ubuntu/nodes/ethereum", + } + GCModeFlag = cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode ("full", "archive")`, + Value: "full", + } +) + +// Parse start and end blocks from command line input +func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { + start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) + if err != nil { + return 0, 0, err + } + end, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) + if err != nil { + return 0, 0, err + } + + return start, end, nil +} + +var localConnections *LocalConnections + +func processAddCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + localConnections.fetchFromNode(start, end) + + localConnections.Chain.Stop() + + return nil +} + +func processShowCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + localConnections.fetchFromNode(start, end) + + localConnections.Chain.Stop() + + return nil +} + +func processVerifyCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + err = setDatabase() + if err != nil { + return fmt.Errorf("Unable to set database connection: %v", err) + } + + verify(start, end) + + localConnections.Chain.Stop() + + return nil +} + +func LDBCLI() { + app := cli.NewApp() + app.Name = filepath.Base(os.Args[0]) + app.Author = "Bugout.dev" + app.Email = "engineering@bugout.dev" + app.Usage = "blockchain ldb extractor command line interface" + app.Flags = []cli.Flag{ + DataDirFlag, + GCModeFlag, + } + app.Commands = []cli.Command{ + { + Name: "add", + Action: utils.MigrateFlags(processAddCommand), + ArgsUsage: " ", + Usage: "Add new blocks with transactions to database", + Description: "This command request blocks from blockchain and adds to database.", + Flags: []cli.Flag{ + DataDirFlag, + GCModeFlag, + }, + }, + { + Name: "show", + Action: utils.MigrateFlags(processShowCommand), + ArgsUsage: " ", + Usage: "Show block with transactions", + Description: "This command print out requested blocks.", + Flags: []cli.Flag{ + DataDirFlag, + GCModeFlag, + }, + }, + { + Name: "verify", + Action: utils.MigrateFlags(processVerifyCommand), + ArgsUsage: " ", + Usage: "Verify blocks with transactions at database", + Description: "This command compare blocks in database and in blockchain for difference.", + Flags: []cli.Flag{ + DataDirFlag, + GCModeFlag, + }, + }, + } + + sort.Sort(cli.FlagsByName(app.Flags)) + sort.Sort(cli.CommandsByName(app.Commands)) + + localConnections = &LocalConnections{} + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go new file mode 100644 index 00000000..a7d17cc8 --- /dev/null +++ b/crawlers/ldb/configs/settings.go @@ -0,0 +1,11 @@ +package configs + +import ( + "os" + "time" +) + +// Database configs +var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 +var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute +var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") diff --git a/crawlers/ldb/go.mod b/crawlers/ldb/go.mod new file mode 100644 index 00000000..3184653c --- /dev/null +++ b/crawlers/ldb/go.mod @@ -0,0 +1,63 @@ +module github.com/bugout-dev/moonstream/crawlers/ldb + +go 1.17 + +require ( + github.com/ethereum/go-ethereum v1.10.16 + github.com/lib/pq v1.0.0 + gopkg.in/urfave/cli.v1 v1.20.0 +) + +require ( + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v1.8.0 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.1.5 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/graph-gophers/graphql-go v1.3.0 // indirect + github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.0 // indirect + github.com/huin/goupnp v1.0.2 // indirect + github.com/influxdata/influxdb v1.8.3 // indirect + github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/tsdb v0.7.1 // indirect + github.com/rjeczalik/notify v0.9.1 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/crawlers/ldb/go.sum b/crawlers/ldb/go.sum new file mode 100644 index 00000000..0512e5eb --- /dev/null +++ b/crawlers/ldb/go.sum @@ -0,0 +1,633 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= +github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/crawlers/ldb/main.go b/crawlers/ldb/main.go new file mode 100644 index 00000000..44a1eda9 --- /dev/null +++ b/crawlers/ldb/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/bugout-dev/moonstream/crawlers/ldb/cmd" +) + +func main() { + cmd.LDBCLI() +} diff --git a/crawlers/ldb/sample.env b/crawlers/ldb/sample.env new file mode 100644 index 00000000..852bb72d --- /dev/null +++ b/crawlers/ldb/sample.env @@ -0,0 +1 @@ +export MOONSTREAM_DB_URI="postgresql://:@:/" From 1642ed9e0a84fc9ecb56a8a36731f5c90f861702 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Mar 2022 21:05:33 +0000 Subject: [PATCH 24/55] Functionality to report at humbug journal with corrupt blocks --- crawlers/ldb/cmd/chain.go | 74 ++++++++++++++++++++++++++------ crawlers/ldb/cmd/cli.go | 31 +++++++++++-- crawlers/ldb/configs/settings.go | 19 ++++++++ crawlers/ldb/go.mod | 3 +- crawlers/ldb/go.sum | 2 + crawlers/ldb/sample.env | 3 ++ 6 files changed, 116 insertions(+), 16 deletions(-) diff --git a/crawlers/ldb/cmd/chain.go b/crawlers/ldb/cmd/chain.go index f530be4a..a3a36615 100644 --- a/crawlers/ldb/cmd/chain.go +++ b/crawlers/ldb/cmd/chain.go @@ -2,11 +2,11 @@ package cmd import ( "database/sql" + "encoding/json" "fmt" - "time" + humbug "github.com/bugout-dev/humbug/go/pkg" "github.com/bugout-dev/moonstream/crawlers/ldb/configs" - _ "github.com/lib/pq" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" + _ "github.com/lib/pq" "gopkg.in/urfave/cli.v1" ) @@ -37,6 +38,10 @@ type LocalConnections struct { Database *sql.DB } +type HumbugReporter struct { + Reporter *humbug.HumbugReporter +} + func setLocalChain(ctx *cli.Context) error { cfg := gethConfig{ Eth: ethconfig.Defaults, @@ -164,31 +169,63 @@ func (lc *LocalConnections) getTxsFromDB(blockNumber uint64) (Transactions, erro return transactions, nil } +type CorruptBlocks struct { + Numbers []uint64 +} + +var corruptBlocks CorruptBlocks + // Write down inconsistent state between database and blockchain -// **source** (string): Source of nonconformity [blockchain, database] -func recordNonconformity(number uint64, source string) { - fmt.Println(number, source) +/* +- number (uint64): Block number +- source (string): Source of nonconformity [blockchain, database] +*/ +func (cb *CorruptBlocks) recordNonconformity(number uint64, source string) { + cb.Numbers = append(cb.Numbers, number) + fmt.Println(cb.Numbers) +} + +func (r *HumbugReporter) submitReport(start, end uint64) error { + content, err := json.Marshal(corruptBlocks) + if err != nil { + return fmt.Errorf("Unable to marshal to json: %v", err) + } + + report := humbug.Report{ + Title: fmt.Sprintf("LDB verifier %d-%d", start, end), + Content: string(content), + Tags: []string{ + fmt.Sprintf("start:%d", start), + fmt.Sprintf("end:%d", end), + }, + } + r.Reporter.Publish(report) + fmt.Println("Error published") + + return nil } func verify(start, end uint64) error { - for i := start; i <= end; i++ { + var cnt uint64 // Counter until report formed and sent to Humbug + + for i := start; i < end; i++ { header, err := localConnections.getBlockFromChain(i) if err != nil { fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) - recordNonconformity(i, "blockchain") + corruptBlocks.recordNonconformity(i, "blockchain") continue } block, err := localConnections.getBlockFromDB(header) if err != nil { fmt.Printf("Unable to get block: %d, err: %v\n", block.BlockNumber, err) - recordNonconformity(block.BlockNumber, "database") + corruptBlocks.recordNonconformity(i, "database") continue } if header.Number.Uint64() != block.BlockNumber { fmt.Printf("Incorrect %d block retrieved from database\n", block.BlockNumber) - recordNonconformity(block.BlockNumber, "database") + corruptBlocks.recordNonconformity(i, "database") continue } @@ -198,18 +235,31 @@ func verify(start, end uint64) error { dbTxs, err := localConnections.getTxsFromDB(header.Number.Uint64()) if err != nil { fmt.Printf("Unable to get transactions: %d, err: %v\n", block.BlockNumber, err) - recordNonconformity(block.BlockNumber, "database") + corruptBlocks.recordNonconformity(i, "database") continue } if chainTxs.Quantity != dbTxs.Quantity { fmt.Printf("Different number of transactions in block %d, err %v\n", block.BlockNumber, err) - recordNonconformity(block.BlockNumber, "database") + corruptBlocks.recordNonconformity(i, "database") continue } fmt.Printf("Processed block number: %d\r", i) - time.Sleep(1 * time.Second) + + cnt++ + if cnt >= configs.BLOCK_RANGE_REPORT { + err := humbugReporter.submitReport(start, end) + if err != nil { + fmt.Printf("Unable to send humbug report: %v", err) + } + cnt = 0 + } + } + + err := humbugReporter.submitReport(start, end) + if err != nil { + fmt.Printf("Unable to send humbug report: %v", err) } fmt.Println("") diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 02445ee0..57e29298 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -8,11 +8,18 @@ import ( "sort" "strconv" + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + + "github.com/bugout-dev/humbug/go/pkg" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/google/uuid" "gopkg.in/urfave/cli.v1" ) var ( + localConnections *LocalConnections + humbugReporter *HumbugReporter + DataDirFlag = cli.StringFlag{ Name: "datadir", Usage: "Data directory for the databases and keystore", @@ -25,6 +32,18 @@ var ( } ) +// Generate humbug client +func setHumbugClient(sessionID string) error { + consent := humbug.CreateHumbugConsent(humbug.True) + reporter, err := humbug.CreateHumbugReporter(consent, configs.HUMBUG_LDB_CLIENT_ID, sessionID, configs.HUMBUG_LDB_TOKEN) + if err != nil { + return fmt.Errorf("Unable to generate humbug reporter: %v", err) + } + humbugReporter.Reporter = reporter + + return nil +} + // Parse start and end blocks from command line input func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) @@ -39,8 +58,6 @@ func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { return start, end, nil } -var localConnections *LocalConnections - func processAddCommand(ctx *cli.Context) error { if ctx.NArg() != 2 { return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) @@ -168,8 +185,16 @@ func LDBCLI() { sort.Sort(cli.CommandsByName(app.Commands)) localConnections = &LocalConnections{} + humbugReporter = &HumbugReporter{} - err := app.Run(os.Args) + // Humbug client to be able write data in Bugout journal + sessionID := uuid.New().String() + err := setHumbugClient(sessionID) + if err != nil { + log.Fatal(err) + } + + err = app.Run(os.Args) if err != nil { log.Fatal(err) } diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go index a7d17cc8..aa8a1872 100644 --- a/crawlers/ldb/configs/settings.go +++ b/crawlers/ldb/configs/settings.go @@ -1,11 +1,30 @@ package configs import ( + "log" "os" + "strconv" "time" ) +var BLOCK_RANGE_REPORT uint64 = 100000 + // Database configs var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") + +// Humber configs +var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID") +var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN") +var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT") + +func init() { + if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" { + ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 0, 16) + if err != nil { + log.Fatal(err) + } + BLOCK_RANGE_REPORT = ldbRangeReport + } +} diff --git a/crawlers/ldb/go.mod b/crawlers/ldb/go.mod index 3184653c..ac48656a 100644 --- a/crawlers/ldb/go.mod +++ b/crawlers/ldb/go.mod @@ -3,7 +3,9 @@ module github.com/bugout-dev/moonstream/crawlers/ldb go 1.17 require ( + github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 github.com/ethereum/go-ethereum v1.10.16 + github.com/google/uuid v1.1.5 github.com/lib/pq v1.0.0 gopkg.in/urfave/cli.v1 v1.20.0 ) @@ -22,7 +24,6 @@ require ( github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.1.5 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/graph-gophers/graphql-go v1.3.0 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect diff --git a/crawlers/ldb/go.sum b/crawlers/ldb/go.sum index 0512e5eb..a708717d 100644 --- a/crawlers/ldb/go.sum +++ b/crawlers/ldb/go.sum @@ -69,6 +69,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 h1:UQ7XGjvoOVKGRIuTFXgqGtU/UgMOk8+ikpoHWrWefjQ= +github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205/go.mod h1:U/NXHfc3tzGeQz+xVfpifXdPZi7p6VV8xdP/4ZKeWJU= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= diff --git a/crawlers/ldb/sample.env b/crawlers/ldb/sample.env index 852bb72d..807ca4ff 100644 --- a/crawlers/ldb/sample.env +++ b/crawlers/ldb/sample.env @@ -1 +1,4 @@ export MOONSTREAM_DB_URI="postgresql://:@:/" +export HUMBUG_LDB_CLIENT_ID="" +export HUMBUG_LDB_TOKEN="" +export HUMBUG_LDB_BLOCK_RANGE_REPORT=100000 From b7578838c8ac462a8a51cb0969992293b1dcdc1f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Mar 2022 21:32:34 +0000 Subject: [PATCH 25/55] Parse uint fix --- crawlers/ldb/configs/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go index aa8a1872..c1e6fd8d 100644 --- a/crawlers/ldb/configs/settings.go +++ b/crawlers/ldb/configs/settings.go @@ -21,7 +21,7 @@ var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT") func init() { if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" { - ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 0, 16) + ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 16, 64) if err != nil { log.Fatal(err) } From ade2bc2c723e1ef31e62ddd95e1aeba859124118 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Fri, 4 Mar 2022 15:50:05 +0300 Subject: [PATCH 26/55] idea in gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 51858600..b3b5f1f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.idea/ .vscode/ .DS_Store \ No newline at end of file From 4085d11226ec559b3b139aeaa83f62b6aa76a96c Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sat, 5 Mar 2022 19:13:42 +0000 Subject: [PATCH 27/55] Respect geth structure due work, optimised sql query to db --- crawlers/ldb/cmd/actions.go | 117 ++++++++++++++ crawlers/ldb/cmd/chain.go | 267 -------------------------------- crawlers/ldb/cmd/cli.go | 27 +--- crawlers/ldb/cmd/connections.go | 145 +++++++++++++++++ crawlers/ldb/cmd/data.go | 54 +++++++ crawlers/ldb/cmd/reporter.go | 45 ++++++ 6 files changed, 366 insertions(+), 289 deletions(-) create mode 100644 crawlers/ldb/cmd/actions.go delete mode 100644 crawlers/ldb/cmd/chain.go create mode 100644 crawlers/ldb/cmd/connections.go create mode 100644 crawlers/ldb/cmd/data.go create mode 100644 crawlers/ldb/cmd/reporter.go diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go new file mode 100644 index 00000000..d60757bf --- /dev/null +++ b/crawlers/ldb/cmd/actions.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "fmt" + + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + + "github.com/ethereum/go-ethereum/common" + _ "github.com/lib/pq" +) + +var ( + // Block which not found in database or have inconsistencies with blockchain + corruptBlocks CorruptBlocks +) + +// Write down inconsistent state between database and blockchain +/* +- number (uint64): Block number +- source (string): Source of nonconformity [blockchain, database] +- description (string): Description of error, why block marked as malformed +*/ +func (cb *CorruptBlocks) registerCorruptBlock(number uint64, source, description string) { + cb.Blocks = append(cb.Blocks, CorruptBlock{ + Number: number, + Source: source, + Description: description, + }) +} + +// Return range of block hashes with transaction hashes from blockchain +func show(start, end uint64) error { + for i := start; i <= end; i++ { + header, err := localConnections.getChainBlock(i) + if err != nil { + fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) + continue + } + + chainTxs := localConnections.getChainTxs(header.Hash(), i) + + var txs []common.Hash + for _, tx := range chainTxs { + txs = append(txs, tx.Hash()) + } + + fmt.Printf("Block %d header with hash: %s and transactions: %s\n", header.Number, header.Hash().String(), txs) + } + + return nil +} + +// Run verification flow of blockchain with database data +func verify(start, end uint64) error { + var cnt uint64 // Counter until report formed and sent to Humbug + + for i := start; i < end; i++ { + header, err := localConnections.getChainBlock(i) + if err != nil { + description := fmt.Sprintf("Unable to get block: %d from chain, err %v", i, err) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "blockchain", description) + continue + } + + dbBlock, err := localConnections.getDatabaseBlockTxs(header.Hash().String()) + + if err != nil { + description := fmt.Sprintf("Unable to get block: %d, err: %v", i, err) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "database", description) + continue + } + + if dbBlock.Number == nil { + description := fmt.Sprintf("Block %d not presented in database", i) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "database", description) + continue + } + + if header.Number.Uint64() != dbBlock.Number.Uint64() { + description := fmt.Sprintf("Incorrect %d block retrieved from database", i) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "database", description) + continue + } + + chainTxs := localConnections.getChainTxs(header.Hash(), i) + + if len(chainTxs) != len(dbBlock.Transactions) { + description := fmt.Sprintf("Different number of transactions in block %d, err %v", i, err) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "database", description) + continue + } + + fmt.Printf("Processed block number: %d\r", i) + + cnt++ + if cnt >= configs.BLOCK_RANGE_REPORT { + err := humbugReporter.submitReport(start, end) + if err != nil { + fmt.Printf("Unable to send humbug report: %v", err) + } + cnt = 0 + } + } + + err := humbugReporter.submitReport(start, end) + if err != nil { + fmt.Printf("Unable to send humbug report: %v", err) + } + fmt.Println("") + + return nil +} diff --git a/crawlers/ldb/cmd/chain.go b/crawlers/ldb/cmd/chain.go deleted file mode 100644 index a3a36615..00000000 --- a/crawlers/ldb/cmd/chain.go +++ /dev/null @@ -1,267 +0,0 @@ -package cmd - -import ( - "database/sql" - "encoding/json" - "fmt" - - humbug "github.com/bugout-dev/humbug/go/pkg" - "github.com/bugout-dev/moonstream/crawlers/ldb/configs" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/node" - _ "github.com/lib/pq" - "gopkg.in/urfave/cli.v1" -) - -type gethConfig struct { - Eth ethconfig.Config - Node node.Config -} - -func defaultNodeConfig() node.Config { - cfg := node.DefaultConfig - cfg.Name = "geth" - return cfg -} - -type LocalConnections struct { - Stack *node.Node - Chain *core.BlockChain - ChainDB ethdb.Database - Database *sql.DB -} - -type HumbugReporter struct { - Reporter *humbug.HumbugReporter -} - -func setLocalChain(ctx *cli.Context) error { - cfg := gethConfig{ - Eth: ethconfig.Defaults, - Node: defaultNodeConfig(), - } - - // Apply flags - utils.SetNodeConfig(ctx, &cfg.Node) - stack, err := node.New(&cfg.Node) - if err != nil { - return fmt.Errorf("Failed to create the protocol stack: %v", err) - } - localConnections.Stack = stack - - utils.SetEthConfig(ctx, stack, &cfg.Eth) - - chain, chainDB := utils.MakeChain(ctx, stack) - localConnections.Chain = chain - localConnections.ChainDB = chainDB - - return nil -} - -func setDatabase() error { - db, err := sql.Open("postgres", configs.MOONSTREAM_DB_URI) - if err != nil { - return fmt.Errorf("DSN parse error or another database initialization error: %v", err) - } - - // Set the maximum number of concurrently idle connections, - // by default sql.DB allows a maximum of 2 idle connections. - db.SetMaxIdleConns(configs.MOONSTREAM_DB_MAX_IDLE_CONNS) - - // Set the maximum lifetime of a connection. - // Longer lifetime increase memory usage. - db.SetConnMaxLifetime(configs.MOONSTREAM_DB_CONN_MAX_LIFETIME) - - localConnections.Database = db - - return nil -} - -func (lc *LocalConnections) fetchFromNode(start, end uint64) error { - for i := start; i <= end; i++ { - header := lc.Chain.GetHeaderByNumber(i) - blockHeaderHash := header.Hash() - body := rawdb.ReadBody(lc.ChainDB, blockHeaderHash, i) - var txs []common.Hash - for _, tx := range body.Transactions { - txs = append(txs, tx.Hash()) - } - - fmt.Printf("- Block header with hash: %s and transactions: %s\n", blockHeaderHash, txs) - } - - return nil -} - -type Block struct { - Hash string `json:"hash"` - BlockNumber uint64 `json:"block_number"` -} - -type Transactions struct { - Txs []string - Quantity int -} - -// Retrive block from blockchain -func (lc *LocalConnections) getBlockFromChain(number uint64) (*types.Header, error) { - header := lc.Chain.GetHeaderByNumber(number) - if header == nil { - return nil, fmt.Errorf("Not found %d block in chain", number) - } - return header, nil -} - -// Retrieve block from database -func (lc *LocalConnections) getBlockFromDB(header *types.Header) (Block, error) { - var block Block - blockRow := lc.Database.QueryRow(fmt.Sprintf("SELECT hash,block_number FROM ethereum_blocks WHERE hash = '%s';", header.Hash().String())) - err := blockRow.Scan(&block.Hash, &block.BlockNumber) - if err != nil { - if err == sql.ErrNoRows { - return block, fmt.Errorf("Not found %d block: %v", header.Number, err) - } - return block, fmt.Errorf("An error occurred during sql operation: %v", err) - } - - return block, nil -} - -// Retrive block transactions from blockchain -func (lc *LocalConnections) getTxsFromChain(headerHash common.Hash, number uint64) Transactions { - var transactions Transactions - body := rawdb.ReadBody(lc.ChainDB, headerHash, number) - for _, tx := range body.Transactions { - transactions.Txs = append(transactions.Txs, tx.Hash().String()) - transactions.Quantity++ - // set[tx.Hash().String()] = false - } - - return transactions -} - -// Retrive block transactions from database -func (lc *LocalConnections) getTxsFromDB(blockNumber uint64) (Transactions, error) { - var transactions Transactions - rows, err := lc.Database.Query(fmt.Sprintf("SELECT hash FROM ethereum_transactions WHERE block_number = %d;", blockNumber)) - if err != nil { - return transactions, fmt.Errorf("An error occurred during sql operation: %v", err) - } - defer rows.Close() - - for rows.Next() { - var txHash string - err := rows.Scan(&txHash) - if err != nil { - return transactions, fmt.Errorf("An error occurred during sql operation: %v", err) - } - transactions.Txs = append(transactions.Txs, txHash) - transactions.Quantity++ - // set[transaction] = true - } - return transactions, nil -} - -type CorruptBlocks struct { - Numbers []uint64 -} - -var corruptBlocks CorruptBlocks - -// Write down inconsistent state between database and blockchain -/* -- number (uint64): Block number -- source (string): Source of nonconformity [blockchain, database] -*/ -func (cb *CorruptBlocks) recordNonconformity(number uint64, source string) { - cb.Numbers = append(cb.Numbers, number) - fmt.Println(cb.Numbers) -} - -func (r *HumbugReporter) submitReport(start, end uint64) error { - content, err := json.Marshal(corruptBlocks) - if err != nil { - return fmt.Errorf("Unable to marshal to json: %v", err) - } - - report := humbug.Report{ - Title: fmt.Sprintf("LDB verifier %d-%d", start, end), - Content: string(content), - Tags: []string{ - fmt.Sprintf("start:%d", start), - fmt.Sprintf("end:%d", end), - }, - } - r.Reporter.Publish(report) - fmt.Println("Error published") - - return nil -} - -func verify(start, end uint64) error { - var cnt uint64 // Counter until report formed and sent to Humbug - - for i := start; i < end; i++ { - header, err := localConnections.getBlockFromChain(i) - if err != nil { - fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) - corruptBlocks.recordNonconformity(i, "blockchain") - continue - } - - block, err := localConnections.getBlockFromDB(header) - if err != nil { - fmt.Printf("Unable to get block: %d, err: %v\n", block.BlockNumber, err) - corruptBlocks.recordNonconformity(i, "database") - continue - } - - if header.Number.Uint64() != block.BlockNumber { - fmt.Printf("Incorrect %d block retrieved from database\n", block.BlockNumber) - corruptBlocks.recordNonconformity(i, "database") - continue - } - - // set := make(map[string]bool) - chainTxs := localConnections.getTxsFromChain(header.Hash(), i) - - dbTxs, err := localConnections.getTxsFromDB(header.Number.Uint64()) - if err != nil { - fmt.Printf("Unable to get transactions: %d, err: %v\n", block.BlockNumber, err) - corruptBlocks.recordNonconformity(i, "database") - continue - } - - if chainTxs.Quantity != dbTxs.Quantity { - fmt.Printf("Different number of transactions in block %d, err %v\n", block.BlockNumber, err) - corruptBlocks.recordNonconformity(i, "database") - continue - } - - fmt.Printf("Processed block number: %d\r", i) - - cnt++ - if cnt >= configs.BLOCK_RANGE_REPORT { - err := humbugReporter.submitReport(start, end) - if err != nil { - fmt.Printf("Unable to send humbug report: %v", err) - } - cnt = 0 - } - } - - err := humbugReporter.submitReport(start, end) - if err != nil { - fmt.Printf("Unable to send humbug report: %v", err) - } - fmt.Println("") - - return nil -} diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 57e29298..4a245061 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -8,18 +8,12 @@ import ( "sort" "strconv" - "github.com/bugout-dev/moonstream/crawlers/ldb/configs" - - "github.com/bugout-dev/humbug/go/pkg" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/google/uuid" "gopkg.in/urfave/cli.v1" ) var ( - localConnections *LocalConnections - humbugReporter *HumbugReporter - DataDirFlag = cli.StringFlag{ Name: "datadir", Usage: "Data directory for the databases and keystore", @@ -32,18 +26,6 @@ var ( } ) -// Generate humbug client -func setHumbugClient(sessionID string) error { - consent := humbug.CreateHumbugConsent(humbug.True) - reporter, err := humbug.CreateHumbugReporter(consent, configs.HUMBUG_LDB_CLIENT_ID, sessionID, configs.HUMBUG_LDB_TOKEN) - if err != nil { - return fmt.Errorf("Unable to generate humbug reporter: %v", err) - } - humbugReporter.Reporter = reporter - - return nil -} - // Parse start and end blocks from command line input func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) @@ -75,7 +57,7 @@ func processAddCommand(ctx *cli.Context) error { defer localConnections.Stack.Close() defer localConnections.ChainDB.Close() - localConnections.fetchFromNode(start, end) + show(start, end) localConnections.Chain.Stop() @@ -99,7 +81,7 @@ func processShowCommand(ctx *cli.Context) error { defer localConnections.Stack.Close() defer localConnections.ChainDB.Close() - localConnections.fetchFromNode(start, end) + show(start, end) localConnections.Chain.Stop() @@ -184,10 +166,11 @@ func LDBCLI() { sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) + // Initialize local connections localConnections = &LocalConnections{} - humbugReporter = &HumbugReporter{} - // Humbug client to be able write data in Bugout journal + // Initialize humbug client to be able write data in Bugout journal + humbugReporter = &HumbugReporter{} sessionID := uuid.New().String() err := setHumbugClient(sessionID) if err != nil { diff --git a/crawlers/ldb/cmd/connections.go b/crawlers/ldb/cmd/connections.go new file mode 100644 index 00000000..a6a66e22 --- /dev/null +++ b/crawlers/ldb/cmd/connections.go @@ -0,0 +1,145 @@ +package cmd + +import ( + "database/sql" + "fmt" + "math/big" + + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" + _ "github.com/lib/pq" + "gopkg.in/urfave/cli.v1" +) + +var ( + localConnections *LocalConnections +) + +// Modified lightweight go-ethereum function +// Source: github.com/ethereum/go-ethereum/cmd/geth/config.go +func defaultNodeConfig() node.Config { + cfg := node.DefaultConfig + cfg.Name = "geth" + return cfg +} + +// Establish connection with blockchain +func setLocalChain(ctx *cli.Context) error { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + } + + // Apply flags + utils.SetNodeConfig(ctx, &cfg.Node) + stack, err := node.New(&cfg.Node) + if err != nil { + return fmt.Errorf("Failed to create the protocol stack: %v", err) + } + localConnections.Stack = stack + + utils.SetEthConfig(ctx, stack, &cfg.Eth) + + chain, chainDB := utils.MakeChain(ctx, stack) + localConnections.Chain = chain + localConnections.ChainDB = chainDB + + return nil +} + +// Establish connection with database +func setDatabase() error { + db, err := sql.Open("postgres", configs.MOONSTREAM_DB_URI) + if err != nil { + return fmt.Errorf("DSN parse error or another database initialization error: %v", err) + } + + // Set the maximum number of concurrently idle connections, + // by default sql.DB allows a maximum of 2 idle connections. + db.SetMaxIdleConns(configs.MOONSTREAM_DB_MAX_IDLE_CONNS) + + // Set the maximum lifetime of a connection. + // Longer lifetime increase memory usage. + db.SetConnMaxLifetime(configs.MOONSTREAM_DB_CONN_MAX_LIFETIME) + + localConnections.Database = db + + return nil +} + +// Retrive block from blockchain +func (lc *LocalConnections) getChainBlock(number uint64) (*types.Header, error) { + header := lc.Chain.GetHeaderByNumber(number) + if header == nil { + return nil, fmt.Errorf("Not found %d block in chain", number) + } + return header, nil +} + +// Retrive block transactions from blockchain +func (lc *LocalConnections) getChainTxs(headerHash common.Hash, number uint64) []*types.Transaction { + var transactions []*types.Transaction + body := rawdb.ReadBody(lc.ChainDB, headerHash, number) + for _, tx := range body.Transactions { + transactions = append(transactions, tx) + } + + return transactions +} + +// Retrive block with transactions from database +func (lc *LocalConnections) getDatabaseBlockTxs(headerHash string) (LightBlock, error) { + var lBlock LightBlock + var txs []LightTransaction + query := fmt.Sprintf( + `SELECT + ethereum_blocks.hash, + ethereum_blocks.block_number, + ethereum_transactions.hash + FROM ethereum_blocks + LEFT JOIN ethereum_transactions ON ethereum_blocks.block_number = ethereum_transactions.block_number + WHERE ethereum_blocks.hash = '%s';`, + headerHash, + ) + rows, err := lc.Database.Query(query) + if err != nil { + return lBlock, fmt.Errorf("An error occurred during sql operation: %v", err) + } + defer rows.Close() + + for rows.Next() { + var blockHash, blockNumberStr, txHash sql.NullString + err := rows.Scan(&blockHash, &blockNumberStr, &txHash) + if err != nil { + return lBlock, fmt.Errorf("An error occurred during sql operation: %v", err) + } + + var lTx LightTransaction + if txHash.Valid != false { + lTx = LightTransaction{ + Hash: txHash.String, + } + txs = append(lBlock.Transactions, lTx) + } + + blockNumber := new(big.Int) + blockNumber, ok := blockNumber.SetString(blockNumberStr.String, 10) + if !ok { + return lBlock, fmt.Errorf("Unable to parse block number") + } + + lBlock = LightBlock{ + Hash: blockHash.String, + Number: blockNumber, + Transactions: txs, + } + } + + return lBlock, nil +} diff --git a/crawlers/ldb/cmd/data.go b/crawlers/ldb/cmd/data.go new file mode 100644 index 00000000..20e7cab8 --- /dev/null +++ b/crawlers/ldb/cmd/data.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "database/sql" + "math/big" + + humbug "github.com/bugout-dev/humbug/go/pkg" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/node" +) + +// Modified lightweight go-ethereum struct +// Source: github.com/ethereum/go-ethereum/cmd/geth/config.go +type gethConfig struct { + Eth ethconfig.Config + Node node.Config +} + +// Predefined connections to blockchain and database +type LocalConnections struct { + Stack *node.Node + Chain *core.BlockChain + ChainDB ethdb.Database + Database *sql.DB +} + +type HumbugReporter struct { + Reporter *humbug.HumbugReporter +} + +// Lightweight transactions for database operations +type LightTransaction struct { + Hash string +} + +// Lightweight block for database operations +type LightBlock struct { + Hash string + Number *big.Int + Transactions []LightTransaction +} + +// Malformed block structure which will be submitted to humbug journal +type CorruptBlock struct { + Number uint64 `json:"number"` + Source string `json:"source"` + Description string `json:"description"` +} + +type CorruptBlocks struct { + Blocks []CorruptBlock `json:"blocks"` +} diff --git a/crawlers/ldb/cmd/reporter.go b/crawlers/ldb/cmd/reporter.go new file mode 100644 index 00000000..2344f6fe --- /dev/null +++ b/crawlers/ldb/cmd/reporter.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "encoding/json" + "fmt" + + "github.com/bugout-dev/humbug/go/pkg" + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" +) + +var ( + humbugReporter *HumbugReporter +) + +// Generate humbug client +func setHumbugClient(sessionID string) error { + consent := humbug.CreateHumbugConsent(humbug.True) + reporter, err := humbug.CreateHumbugReporter(consent, configs.HUMBUG_LDB_CLIENT_ID, sessionID, configs.HUMBUG_LDB_TOKEN) + if err != nil { + return fmt.Errorf("Unable to generate humbug reporter: %v", err) + } + humbugReporter.Reporter = reporter + + return nil +} + +func (r *HumbugReporter) submitReport(start, end uint64) error { + content, err := json.Marshal(corruptBlocks) + if err != nil { + return fmt.Errorf("Unable to marshal to json: %v", err) + } + + report := humbug.Report{ + Title: fmt.Sprintf("LDB verifier %d-%d", start, end), + Content: string(content), + Tags: []string{ + fmt.Sprintf("start:%d", start), + fmt.Sprintf("end:%d", end), + }, + } + r.Reporter.Publish(report) + fmt.Println("Error published") + + return nil +} From e8ae125dd16c4d2c705b950e59da590b2e0fd784 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sat, 5 Mar 2022 19:49:53 +0000 Subject: [PATCH 28/55] Blockchain flag --- crawlers/ldb/cmd/actions.go | 12 ++- crawlers/ldb/cmd/cli.go | 132 ++++++++++++++++++++------------ crawlers/ldb/cmd/connections.go | 22 ++++-- 3 files changed, 107 insertions(+), 59 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index d60757bf..2f04709c 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -28,6 +28,10 @@ func (cb *CorruptBlocks) registerCorruptBlock(number uint64, source, description }) } +func add(blockchain string, start, end uint64) error { + return nil +} + // Return range of block hashes with transaction hashes from blockchain func show(start, end uint64) error { for i := start; i <= end; i++ { @@ -51,7 +55,7 @@ func show(start, end uint64) error { } // Run verification flow of blockchain with database data -func verify(start, end uint64) error { +func verify(blockchain string, start, end uint64) error { var cnt uint64 // Counter until report formed and sent to Humbug for i := start; i < end; i++ { @@ -63,7 +67,7 @@ func verify(start, end uint64) error { continue } - dbBlock, err := localConnections.getDatabaseBlockTxs(header.Hash().String()) + dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, header.Hash().String()) if err != nil { description := fmt.Sprintf("Unable to get block: %d, err: %v", i, err) @@ -101,7 +105,7 @@ func verify(start, end uint64) error { if cnt >= configs.BLOCK_RANGE_REPORT { err := humbugReporter.submitReport(start, end) if err != nil { - fmt.Printf("Unable to send humbug report: %v", err) + return fmt.Errorf("Unable to send humbug report: %v", err) } cnt = 0 } @@ -109,7 +113,7 @@ func verify(start, end uint64) error { err := humbugReporter.submitReport(start, end) if err != nil { - fmt.Printf("Unable to send humbug report: %v", err) + return fmt.Errorf("Unable to send humbug report: %v", err) } fmt.Println("") diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 4a245061..af79cfc8 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -14,6 +14,10 @@ import ( ) var ( + BlockchainFlag = cli.StringFlag{ + Name: "blockchain", + Usage: `Which blockchain to crawl ("ethereum", "polygon")`, + } DataDirFlag = cli.StringFlag{ Name: "datadir", Usage: "Data directory for the databases and keystore", @@ -27,6 +31,7 @@ var ( ) // Parse start and end blocks from command line input +// TODO(kompotkot): Re-write to work via channel in goroutines func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) if err != nil { @@ -44,53 +49,9 @@ func processAddCommand(ctx *cli.Context) error { if ctx.NArg() != 2 { return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) } - - start, end, err := startEndBlock(ctx) - if err != nil { - return fmt.Errorf("Unable to parse block range: %v", err) - } - - err = setLocalChain(ctx) - if err != nil { - return fmt.Errorf("Unable to set blockchain: %v", err) - } - defer localConnections.Stack.Close() - defer localConnections.ChainDB.Close() - - show(start, end) - - localConnections.Chain.Stop() - - return nil -} - -func processShowCommand(ctx *cli.Context) error { - if ctx.NArg() != 2 { - return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) - } - - start, end, err := startEndBlock(ctx) - if err != nil { - return fmt.Errorf("Unable to parse block range: %v", err) - } - - err = setLocalChain(ctx) - if err != nil { - return fmt.Errorf("Unable to set blockchain: %v", err) - } - defer localConnections.Stack.Close() - defer localConnections.ChainDB.Close() - - show(start, end) - - localConnections.Chain.Stop() - - return nil -} - -func processVerifyCommand(ctx *cli.Context) error { - if ctx.NArg() != 2 { - return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") } start, end, err := startEndBlock(ctx) @@ -110,7 +71,77 @@ func processVerifyCommand(ctx *cli.Context) error { return fmt.Errorf("Unable to set database connection: %v", err) } - verify(start, end) + err = add(blockchain, start, end) + if err != nil { + return fmt.Errorf("Error occurred due add acction: %v", err) + } + + localConnections.Chain.Stop() + + return nil +} + +func processShowCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + err = show(start, end) + if err != nil { + return fmt.Errorf("Error occurred due show acction: %v", err) + } + + localConnections.Chain.Stop() + + return nil +} + +func processVerifyCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + err = setDatabase() + if err != nil { + return fmt.Errorf("Unable to set database connection: %v", err) + } + + err = verify(blockchain, start, end) + if err != nil { + return fmt.Errorf("Error occurred due verify acction: %v", err) + } localConnections.Chain.Stop() @@ -124,9 +155,11 @@ func LDBCLI() { app.Email = "engineering@bugout.dev" app.Usage = "blockchain ldb extractor command line interface" app.Flags = []cli.Flag{ + BlockchainFlag, DataDirFlag, GCModeFlag, } + app.Commands = []cli.Command{ { Name: "add", @@ -135,6 +168,7 @@ func LDBCLI() { Usage: "Add new blocks with transactions to database", Description: "This command request blocks from blockchain and adds to database.", Flags: []cli.Flag{ + BlockchainFlag, DataDirFlag, GCModeFlag, }, @@ -146,6 +180,7 @@ func LDBCLI() { Usage: "Show block with transactions", Description: "This command print out requested blocks.", Flags: []cli.Flag{ + BlockchainFlag, DataDirFlag, GCModeFlag, }, @@ -157,6 +192,7 @@ func LDBCLI() { Usage: "Verify blocks with transactions at database", Description: "This command compare blocks in database and in blockchain for difference.", Flags: []cli.Flag{ + BlockchainFlag, DataDirFlag, GCModeFlag, }, diff --git a/crawlers/ldb/cmd/connections.go b/crawlers/ldb/cmd/connections.go index a6a66e22..4fdd1c18 100644 --- a/crawlers/ldb/cmd/connections.go +++ b/crawlers/ldb/cmd/connections.go @@ -94,17 +94,25 @@ func (lc *LocalConnections) getChainTxs(headerHash common.Hash, number uint64) [ } // Retrive block with transactions from database -func (lc *LocalConnections) getDatabaseBlockTxs(headerHash string) (LightBlock, error) { +func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, headerHash string) (LightBlock, error) { var lBlock LightBlock var txs []LightTransaction query := fmt.Sprintf( `SELECT - ethereum_blocks.hash, - ethereum_blocks.block_number, - ethereum_transactions.hash - FROM ethereum_blocks - LEFT JOIN ethereum_transactions ON ethereum_blocks.block_number = ethereum_transactions.block_number - WHERE ethereum_blocks.hash = '%s';`, + %s_blocks.hash, + %s_blocks.block_number, + %s_transactions.hash + FROM %s_blocks + LEFT JOIN %s_transactions ON %s_blocks.block_number = %s_transactions.block_number + WHERE %s_blocks.hash = '%s';`, + blockchain, + blockchain, + blockchain, + blockchain, + blockchain, + blockchain, + blockchain, + blockchain, headerHash, ) rows, err := lc.Database.Query(query) From 392bdf801d7817d67eb9b9c81de224bfe30fb96f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 00:11:13 +0000 Subject: [PATCH 29/55] Insert block --- crawlers/ldb/cmd/actions.go | 57 +++++++++++++------- crawlers/ldb/cmd/connections.go | 93 +++++++++++++++++++++++++++------ 2 files changed, 115 insertions(+), 35 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 2f04709c..8c643a1a 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -3,7 +3,7 @@ package cmd import ( "fmt" - "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + // "github.com/bugout-dev/moonstream/crawlers/ldb/configs" "github.com/ethereum/go-ethereum/common" _ "github.com/lib/pq" @@ -29,26 +29,45 @@ func (cb *CorruptBlocks) registerCorruptBlock(number uint64, source, description } func add(blockchain string, start, end uint64) error { + for i := start; i < end; i++ { + block, err := localConnections.getChainBlock(i) + if err != nil { + description := fmt.Sprintf("Unable to get block: %d from chain, err %v", i, err) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(i, "blockchain", description) + continue + } + td := localConnections.Chain.GetTd(block.Hash(), block.NumberU64()) + + chainTxs := localConnections.getChainTxs(block.Hash(), i) + + err = localConnections.writeDatabaseBlockTxs(blockchain, block, chainTxs, td) + if err != nil { + fmt.Printf("Error occurred due saving block %d with transactions in database: %v", i, err) + } + + fmt.Printf("Processed block number: %d\r", i) + } return nil } // Return range of block hashes with transaction hashes from blockchain func show(start, end uint64) error { for i := start; i <= end; i++ { - header, err := localConnections.getChainBlock(i) + block, err := localConnections.getChainBlock(i) if err != nil { fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) continue } - chainTxs := localConnections.getChainTxs(header.Hash(), i) + chainTxs := localConnections.getChainTxs(block.Hash(), i) var txs []common.Hash for _, tx := range chainTxs { txs = append(txs, tx.Hash()) } - fmt.Printf("Block %d header with hash: %s and transactions: %s\n", header.Number, header.Hash().String(), txs) + fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs) } return nil @@ -59,7 +78,7 @@ func verify(blockchain string, start, end uint64) error { var cnt uint64 // Counter until report formed and sent to Humbug for i := start; i < end; i++ { - header, err := localConnections.getChainBlock(i) + block, err := localConnections.getChainBlock(i) if err != nil { description := fmt.Sprintf("Unable to get block: %d from chain, err %v", i, err) fmt.Println(description) @@ -67,7 +86,7 @@ func verify(blockchain string, start, end uint64) error { continue } - dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, header.Hash().String()) + dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, block.Hash().String()) if err != nil { description := fmt.Sprintf("Unable to get block: %d, err: %v", i, err) @@ -83,14 +102,14 @@ func verify(blockchain string, start, end uint64) error { continue } - if header.Number.Uint64() != dbBlock.Number.Uint64() { + if block.NumberU64() != dbBlock.Number.Uint64() { description := fmt.Sprintf("Incorrect %d block retrieved from database", i) fmt.Println(description) corruptBlocks.registerCorruptBlock(i, "database", description) continue } - chainTxs := localConnections.getChainTxs(header.Hash(), i) + chainTxs := localConnections.getChainTxs(block.Hash(), i) if len(chainTxs) != len(dbBlock.Transactions) { description := fmt.Sprintf("Different number of transactions in block %d, err %v", i, err) @@ -102,19 +121,19 @@ func verify(blockchain string, start, end uint64) error { fmt.Printf("Processed block number: %d\r", i) cnt++ - if cnt >= configs.BLOCK_RANGE_REPORT { - err := humbugReporter.submitReport(start, end) - if err != nil { - return fmt.Errorf("Unable to send humbug report: %v", err) - } - cnt = 0 - } + // if cnt >= configs.BLOCK_RANGE_REPORT { + // err := humbugReporter.submitReport(start, end) + // if err != nil { + // return fmt.Errorf("Unable to send humbug report: %v", err) + // } + // cnt = 0 + // } } - err := humbugReporter.submitReport(start, end) - if err != nil { - return fmt.Errorf("Unable to send humbug report: %v", err) - } + // err := humbugReporter.submitReport(start, end) + // if err != nil { + // return fmt.Errorf("Unable to send humbug report: %v", err) + // } fmt.Println("") return nil diff --git a/crawlers/ldb/cmd/connections.go b/crawlers/ldb/cmd/connections.go index 4fdd1c18..6a91462e 100644 --- a/crawlers/ldb/cmd/connections.go +++ b/crawlers/ldb/cmd/connections.go @@ -74,18 +74,18 @@ func setDatabase() error { } // Retrive block from blockchain -func (lc *LocalConnections) getChainBlock(number uint64) (*types.Header, error) { - header := lc.Chain.GetHeaderByNumber(number) - if header == nil { +func (lc *LocalConnections) getChainBlock(number uint64) (*types.Block, error) { + block := lc.Chain.GetBlockByNumber(number) + if block == nil { return nil, fmt.Errorf("Not found %d block in chain", number) } - return header, nil + return block, nil } // Retrive block transactions from blockchain -func (lc *LocalConnections) getChainTxs(headerHash common.Hash, number uint64) []*types.Transaction { +func (lc *LocalConnections) getChainTxs(blockHash common.Hash, number uint64) []*types.Transaction { var transactions []*types.Transaction - body := rawdb.ReadBody(lc.ChainDB, headerHash, number) + body := rawdb.ReadBody(lc.ChainDB, blockHash, number) for _, tx := range body.Transactions { transactions = append(transactions, tx) } @@ -94,7 +94,7 @@ func (lc *LocalConnections) getChainTxs(headerHash common.Hash, number uint64) [ } // Retrive block with transactions from database -func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, headerHash string) (LightBlock, error) { +func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, blockHash string) (LightBlock, error) { var lBlock LightBlock var txs []LightTransaction query := fmt.Sprintf( @@ -105,15 +105,7 @@ func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, headerHash string) ( FROM %s_blocks LEFT JOIN %s_transactions ON %s_blocks.block_number = %s_transactions.block_number WHERE %s_blocks.hash = '%s';`, - blockchain, - blockchain, - blockchain, - blockchain, - blockchain, - blockchain, - blockchain, - blockchain, - headerHash, + blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockHash, ) rows, err := lc.Database.Query(query) if err != nil { @@ -151,3 +143,72 @@ func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, headerHash string) ( return lBlock, nil } + +// Write block with transactions to database +func (lc *LocalConnections) writeDatabaseBlockTxs( + blockchain string, block *types.Block, txs []*types.Transaction, td *big.Int, +) error { + // block.extraData doesn't exist at Polygon mainnet + var extraData interface{} + if block.Extra() == nil { + extraData = "NULL" + } else { + extraData = fmt.Sprintf("'0x%x'", block.Extra()) + } + + // For support of blocks before London hardfork + var baseFee interface{} + if block.BaseFee() == nil { + baseFee = "NULL" + } else { + baseFee = block.BaseFee() + } + + query := fmt.Sprintf( + `INSERT INTO ethereum_blocks ( + block_number, + difficulty, + extra_data, + gas_limit, + gas_used, + base_fee_per_gas, + hash, + logs_bloom, + miner, + nonce, + parent_hash, + receipt_root, + uncles, + size, + state_root, + timestamp, + total_difficulty, + transactions_root + ) + VALUES (%d, %d, %v, %d, %d, %v, '%s', '0x%x', '%s', '0x%x', '0x%x', '0x%x', '0x%x', %f, '0x%x', %d, %d, '0x%x');`, + block.Number(), + block.Difficulty(), + extraData, + block.GasLimit(), + block.GasUsed(), + baseFee, + block.Hash(), + block.Bloom(), + block.Coinbase(), + block.Nonce(), + block.ParentHash(), + block.ReceiptHash(), + block.UncleHash(), + block.Size(), + block.Root(), + block.Time(), + td, + block.TxHash(), + ) + _, err := lc.Database.Exec(query) + if err != nil { + return fmt.Errorf("An error occurred during sql operation: %v", err) + } + + return nil +} From cb8bb515aa40808725caceb16aa84de6a1beaac4 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 18:10:55 +0000 Subject: [PATCH 30/55] Blocks range generator --- crawlers/ldb/cmd/actions.go | 79 +++++++++++++++------------------ crawlers/ldb/cmd/cli.go | 74 +++++++++++++++++++++++++----- crawlers/ldb/cmd/connections.go | 43 +++++++++++++++--- crawlers/ldb/cmd/reporter.go | 2 +- 4 files changed, 137 insertions(+), 61 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 8c643a1a..9345c43d 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -3,8 +3,6 @@ package cmd import ( "fmt" - // "github.com/bugout-dev/moonstream/crawlers/ldb/configs" - "github.com/ethereum/go-ethereum/common" _ "github.com/lib/pq" ) @@ -28,39 +26,40 @@ func (cb *CorruptBlocks) registerCorruptBlock(number uint64, source, description }) } -func add(blockchain string, start, end uint64) error { - for i := start; i < end; i++ { - block, err := localConnections.getChainBlock(i) +// Add new blocks with transactions to database +func add(blockchain string, blockNumbers []uint64) error { + for _, bn := range blockNumbers { + block, err := localConnections.getChainBlock(bn) if err != nil { - description := fmt.Sprintf("Unable to get block: %d from chain, err %v", i, err) + description := fmt.Sprintf("Unable to get block: %d from chain, err %v", bn, err) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "blockchain", description) + corruptBlocks.registerCorruptBlock(bn, "blockchain", description) continue } td := localConnections.Chain.GetTd(block.Hash(), block.NumberU64()) - chainTxs := localConnections.getChainTxs(block.Hash(), i) + chainTxs := localConnections.getChainTxs(block.Hash(), bn) err = localConnections.writeDatabaseBlockTxs(blockchain, block, chainTxs, td) if err != nil { - fmt.Printf("Error occurred due saving block %d with transactions in database: %v", i, err) + fmt.Printf("Error occurred due saving block %d with transactions in database: %v", bn, err) } - fmt.Printf("Processed block number: %d\r", i) + fmt.Printf("Processed block number: %d\r", bn) } return nil } // Return range of block hashes with transaction hashes from blockchain -func show(start, end uint64) error { - for i := start; i <= end; i++ { - block, err := localConnections.getChainBlock(i) +func show(blockNumbers []uint64) error { + for _, bn := range blockNumbers { + block, err := localConnections.getChainBlock(bn) if err != nil { - fmt.Printf("Unable to get block: %d from chain, err %v\n", i, err) + fmt.Printf("Unable to get block: %d from chain, err %v\n", bn, err) continue } - chainTxs := localConnections.getChainTxs(block.Hash(), i) + chainTxs := localConnections.getChainTxs(block.Hash(), bn) var txs []common.Hash for _, tx := range chainTxs { @@ -74,67 +73,59 @@ func show(start, end uint64) error { } // Run verification flow of blockchain with database data -func verify(blockchain string, start, end uint64) error { +func verify(blockchain string, blockNumbers []uint64) error { var cnt uint64 // Counter until report formed and sent to Humbug - for i := start; i < end; i++ { - block, err := localConnections.getChainBlock(i) + for _, bn := range blockNumbers { + chainBlock, err := localConnections.getChainBlock(bn) if err != nil { - description := fmt.Sprintf("Unable to get block: %d from chain, err %v", i, err) + description := fmt.Sprintf("Unable to get block: %d from chain, err %v", bn, err) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "blockchain", description) + corruptBlocks.registerCorruptBlock(bn, "blockchain", description) continue } - dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, block.Hash().String()) + dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String()) if err != nil { - description := fmt.Sprintf("Unable to get block: %d, err: %v", i, err) + description := fmt.Sprintf("Unable to get block: %d, err: %v", bn, err) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "database", description) + corruptBlocks.registerCorruptBlock(bn, "database", description) continue } if dbBlock.Number == nil { - description := fmt.Sprintf("Block %d not presented in database", i) + description := fmt.Sprintf("Block %d not presented in database", bn) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "database", description) + corruptBlocks.registerCorruptBlock(bn, "database", description) continue } - if block.NumberU64() != dbBlock.Number.Uint64() { - description := fmt.Sprintf("Incorrect %d block retrieved from database", i) + if chainBlock.NumberU64() != dbBlock.Number.Uint64() { + description := fmt.Sprintf("Incorrect %d block retrieved from database", bn) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "database", description) + corruptBlocks.registerCorruptBlock(bn, "database", description) continue } - chainTxs := localConnections.getChainTxs(block.Hash(), i) + chainTxs := localConnections.getChainTxs(chainBlock.Hash(), bn) if len(chainTxs) != len(dbBlock.Transactions) { - description := fmt.Sprintf("Different number of transactions in block %d, err %v", i, err) + description := fmt.Sprintf("Different number of transactions in block %d, err %v", bn, err) fmt.Println(description) - corruptBlocks.registerCorruptBlock(i, "database", description) + corruptBlocks.registerCorruptBlock(bn, "database", description) continue } - fmt.Printf("Processed block number: %d\r", i) + fmt.Printf("Processed block number: %d\r", bn) cnt++ - // if cnt >= configs.BLOCK_RANGE_REPORT { - // err := humbugReporter.submitReport(start, end) - // if err != nil { - // return fmt.Errorf("Unable to send humbug report: %v", err) - // } - // cnt = 0 - // } } - // err := humbugReporter.submitReport(start, end) - // if err != nil { - // return fmt.Errorf("Unable to send humbug report: %v", err) - // } - fmt.Println("") + err := humbugReporter.submitReport(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + if err != nil { + return fmt.Errorf("Unable to send humbug report: %v", err) + } return nil } diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index af79cfc8..fdcb9388 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -14,6 +14,8 @@ import ( ) var ( + BlockNumberStep = uint64(1000) + BlockchainFlag = cli.StringFlag{ Name: "blockchain", Usage: `Which blockchain to crawl ("ethereum", "polygon")`, @@ -30,18 +32,62 @@ var ( } ) +// Block step generator, yield list of blocks with length equal blockStep +func BlockYield(start, end, blockStep uint64) chan []uint64 { + ch := make(chan []uint64) + + go func(ch chan []uint64) { + currentBlock := start + var tempEnd uint64 + + for { + if currentBlock >= end { + break + } + + tempEnd = currentBlock + blockStep + if tempEnd >= end { + tempEnd = end + } + + var blocks []uint64 + for i := currentBlock; i < tempEnd; i++ { + blocks = append(blocks, i) // Block operation + } + ch <- blocks + + currentBlock += blockStep + } + + close(ch) + }(ch) + + return ch +} + // Parse start and end blocks from command line input // TODO(kompotkot): Re-write to work via channel in goroutines func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { - start, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) + inputStart, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) if err != nil { return 0, 0, err } - end, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) + inputEnd, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) if err != nil { return 0, 0, err } + var start, end uint64 + if inputStart < inputEnd { + start = inputStart + end = inputEnd + } else if inputStart > inputEnd { + start = inputEnd + end = inputStart + } else { + return 0, 0, fmt.Errorf("Start and end block can't be equal") + } + return start, end, nil } @@ -71,9 +117,11 @@ func processAddCommand(ctx *cli.Context) error { return fmt.Errorf("Unable to set database connection: %v", err) } - err = add(blockchain, start, end) - if err != nil { - return fmt.Errorf("Error occurred due add acction: %v", err) + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = add(blockchain, blocks) + if err != nil { + return fmt.Errorf("Error occurred due add acction: %v", err) + } } localConnections.Chain.Stop() @@ -102,9 +150,11 @@ func processShowCommand(ctx *cli.Context) error { defer localConnections.Stack.Close() defer localConnections.ChainDB.Close() - err = show(start, end) - if err != nil { - return fmt.Errorf("Error occurred due show acction: %v", err) + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = show(blocks) + if err != nil { + return fmt.Errorf("Error occurred due show acction: %v", err) + } } localConnections.Chain.Stop() @@ -138,9 +188,11 @@ func processVerifyCommand(ctx *cli.Context) error { return fmt.Errorf("Unable to set database connection: %v", err) } - err = verify(blockchain, start, end) - if err != nil { - return fmt.Errorf("Error occurred due verify acction: %v", err) + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = verify(blockchain, blocks) + if err != nil { + return fmt.Errorf("Error occurred due verify acction: %v", err) + } } localConnections.Chain.Stop() diff --git a/crawlers/ldb/cmd/connections.go b/crawlers/ldb/cmd/connections.go index 6a91462e..ab34a542 100644 --- a/crawlers/ldb/cmd/connections.go +++ b/crawlers/ldb/cmd/connections.go @@ -164,8 +164,8 @@ func (lc *LocalConnections) writeDatabaseBlockTxs( baseFee = block.BaseFee() } - query := fmt.Sprintf( - `INSERT INTO ethereum_blocks ( + blockQuery := fmt.Sprintf( + `INSERT INTO %s_blocks ( block_number, difficulty, extra_data, @@ -184,8 +184,8 @@ func (lc *LocalConnections) writeDatabaseBlockTxs( timestamp, total_difficulty, transactions_root - ) - VALUES (%d, %d, %v, %d, %d, %v, '%s', '0x%x', '%s', '0x%x', '0x%x', '0x%x', '0x%x', %f, '0x%x', %d, %d, '0x%x');`, + ) VALUES (%d, %d, %v, %d, %d, %v, '%s', '0x%x', '%s', '0x%x', '0x%x', '0x%x', '0x%x', %f, '0x%x', %d, %d, '0x%x');`, + blockchain, block.Number(), block.Difficulty(), extraData, @@ -205,10 +205,43 @@ func (lc *LocalConnections) writeDatabaseBlockTxs( td, block.TxHash(), ) - _, err := lc.Database.Exec(query) + _, err := lc.Database.Exec(blockQuery) if err != nil { return fmt.Errorf("An error occurred during sql operation: %v", err) } + // for _, tx := range(txs) { + // txQuery := fmt.Sprintf( + // `INSERT INTO %s_transactions ( + // hash, + // block_number, + // from_address, + // to_address, + // gas, + // gas_price, + // max_fee_per_gas, + // max_priority_fee_per_gas, + // input, + // nonce, + // transaction_index, + // transaction_type, + // value + // ) VALUES ('%s', %d, '%s', '%s', %d, %d, );`, + // blockchain, + // tx.Hash(), + // block.Number(), + // tx.from, + // tx.To(), + // tx.Gas(), + // tx.GasPrice(), + // "max_fee", + // "max_prior", + // tx.input, + // tx.Nonce(), + // "tx_index", + // tx.Type(), + // tx.Value(), + // ) + // } return nil } diff --git a/crawlers/ldb/cmd/reporter.go b/crawlers/ldb/cmd/reporter.go index 2344f6fe..25e3ab5b 100644 --- a/crawlers/ldb/cmd/reporter.go +++ b/crawlers/ldb/cmd/reporter.go @@ -39,7 +39,7 @@ func (r *HumbugReporter) submitReport(start, end uint64) error { }, } r.Reporter.Publish(report) - fmt.Println("Error published") + fmt.Printf("Error for range %d-%d published\n", start, end) return nil } From c33d033ad9e0514116a207ad0932a5e91bd25827 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 18:51:36 +0000 Subject: [PATCH 31/55] Smart verification humbug report counter --- crawlers/ldb/cmd/actions.go | 16 +++++++--------- crawlers/ldb/cmd/cli.go | 19 +++++++++++++++++++ crawlers/ldb/cmd/reporter.go | 7 ++++--- crawlers/ldb/configs/settings.go | 13 ------------- crawlers/ldb/sample.env | 1 - 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 9345c43d..5c89ebc0 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -47,6 +47,7 @@ func add(blockchain string, blockNumbers []uint64) error { fmt.Printf("Processed block number: %d\r", bn) } + return nil } @@ -59,16 +60,18 @@ func show(blockNumbers []uint64) error { continue } - chainTxs := localConnections.getChainTxs(block.Hash(), bn) + // chainTxs := localConnections.getChainTxs(block.Hash(), bn) var txs []common.Hash - for _, tx := range chainTxs { - txs = append(txs, tx.Hash()) - } + // for _, tx := range chainTxs { + // txs = append(txs, tx.Hash()) + // } fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs) } + fmt.Println("new range") + return nil } @@ -122,10 +125,5 @@ func verify(blockchain string, blockNumbers []uint64) error { cnt++ } - err := humbugReporter.submitReport(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) - if err != nil { - return fmt.Errorf("Unable to send humbug report: %v", err) - } - return nil } diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index fdcb9388..89707fea 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -8,12 +8,14 @@ import ( "sort" "strconv" + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/google/uuid" "gopkg.in/urfave/cli.v1" ) var ( + // Block steps used to prevent long executor tasks and data loss possibility BlockNumberStep = uint64(1000) BlockchainFlag = cli.StringFlag{ @@ -188,11 +190,28 @@ func processVerifyCommand(ctx *cli.Context) error { return fmt.Errorf("Unable to set database connection: %v", err) } + cnt := uint64(0) + reportStart := uint64(start) for blocks := range BlockYield(start, end, BlockNumberStep) { err = verify(blockchain, blocks) if err != nil { return fmt.Errorf("Error occurred due verify acction: %v", err) } + + cnt += BlockNumberStep + if cnt >= configs.BLOCK_RANGE_REPORT { + err := humbugReporter.submitReport(reportStart, blocks[len(blocks)-1]+1, "") + if err != nil { + return fmt.Errorf("Unable to send humbug report: %v", err) + } + reportStart = blocks[len(blocks)-1] + 1 + cnt = 0 + } + } + + err = humbugReporter.submitReport(start, end, "Total ") + if err != nil { + return fmt.Errorf("Unable to send humbug report: %v", err) } localConnections.Chain.Stop() diff --git a/crawlers/ldb/cmd/reporter.go b/crawlers/ldb/cmd/reporter.go index 25e3ab5b..bd4e2cd2 100644 --- a/crawlers/ldb/cmd/reporter.go +++ b/crawlers/ldb/cmd/reporter.go @@ -24,14 +24,15 @@ func setHumbugClient(sessionID string) error { return nil } -func (r *HumbugReporter) submitReport(start, end uint64) error { +// Publish report with verified blocks to entry +func (r *HumbugReporter) submitReport(start, end uint64, prefix string) error { content, err := json.Marshal(corruptBlocks) if err != nil { return fmt.Errorf("Unable to marshal to json: %v", err) } report := humbug.Report{ - Title: fmt.Sprintf("LDB verifier %d-%d", start, end), + Title: fmt.Sprintf("%sLDB verifier %d-%d", prefix, start, end), Content: string(content), Tags: []string{ fmt.Sprintf("start:%d", start), @@ -39,7 +40,7 @@ func (r *HumbugReporter) submitReport(start, end uint64) error { }, } r.Reporter.Publish(report) - fmt.Printf("Error for range %d-%d published\n", start, end) + fmt.Printf("%sLDB report for range %d-%d published\n", prefix, start, end) return nil } diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go index c1e6fd8d..19914539 100644 --- a/crawlers/ldb/configs/settings.go +++ b/crawlers/ldb/configs/settings.go @@ -1,9 +1,7 @@ package configs import ( - "log" "os" - "strconv" "time" ) @@ -17,14 +15,3 @@ var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") // Humber configs var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID") var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN") -var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT") - -func init() { - if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" { - ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 16, 64) - if err != nil { - log.Fatal(err) - } - BLOCK_RANGE_REPORT = ldbRangeReport - } -} diff --git a/crawlers/ldb/sample.env b/crawlers/ldb/sample.env index 807ca4ff..449b363a 100644 --- a/crawlers/ldb/sample.env +++ b/crawlers/ldb/sample.env @@ -1,4 +1,3 @@ export MOONSTREAM_DB_URI="postgresql://:@:/" export HUMBUG_LDB_CLIENT_ID="" export HUMBUG_LDB_TOKEN="" -export HUMBUG_LDB_BLOCK_RANGE_REPORT=100000 From 39c391dfa026652793ff2fde627251d9e775d085 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 19:04:01 +0000 Subject: [PATCH 32/55] Final 1 thread version --- crawlers/ldb/cmd/actions.go | 14 ++++---------- crawlers/ldb/cmd/cli.go | 1 + 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 5c89ebc0..85c89389 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -60,25 +60,21 @@ func show(blockNumbers []uint64) error { continue } - // chainTxs := localConnections.getChainTxs(block.Hash(), bn) + chainTxs := localConnections.getChainTxs(block.Hash(), bn) var txs []common.Hash - // for _, tx := range chainTxs { - // txs = append(txs, tx.Hash()) - // } + for _, tx := range chainTxs { + txs = append(txs, tx.Hash()) + } fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs) } - fmt.Println("new range") - return nil } // Run verification flow of blockchain with database data func verify(blockchain string, blockNumbers []uint64) error { - var cnt uint64 // Counter until report formed and sent to Humbug - for _, bn := range blockNumbers { chainBlock, err := localConnections.getChainBlock(bn) if err != nil { @@ -121,8 +117,6 @@ func verify(blockchain string, blockNumbers []uint64) error { } fmt.Printf("Processed block number: %d\r", bn) - - cnt++ } return nil diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 89707fea..8f43b267 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -9,6 +9,7 @@ import ( "strconv" "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/google/uuid" "gopkg.in/urfave/cli.v1" From 838c75ae294c7a7b8d3604501a01bdc7ef6eb2f6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 22:59:47 +0000 Subject: [PATCH 33/55] Verification command in goroutines --- crawlers/ldb/cmd/actions.go | 137 +++++++++++++++++++++++-------- crawlers/ldb/cmd/cli.go | 28 +++---- crawlers/ldb/configs/settings.go | 2 - 3 files changed, 116 insertions(+), 51 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 85c89389..5e462306 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -73,50 +73,117 @@ func show(blockNumbers []uint64) error { return nil } +// TODO(kompotkot): Find way to remove Number +type Result struct { + ErrorOutput string + ErrorSource string + Number uint64 + Output string +} + +type Job struct { + BlockNumber uint64 + Results chan<- Result +} + // Run verification flow of blockchain with database data -func verify(blockchain string, blockNumbers []uint64) error { - for _, bn := range blockNumbers { - chainBlock, err := localConnections.getChainBlock(bn) - if err != nil { - description := fmt.Sprintf("Unable to get block: %d from chain, err %v", bn, err) - fmt.Println(description) - corruptBlocks.registerCorruptBlock(bn, "blockchain", description) - continue +func verify(blockchain string, blockNumbers []uint64, workers int) error { + jobsCh := make(chan Job, workers) + resultCh := make(chan Result, len(blockNumbers)) + doneCh := make(chan struct{}, workers) + + // Add jobs + go func() { + for _, bn := range blockNumbers { + jobsCh <- Job{ + BlockNumber: bn, + Results: resultCh, + } } + close(jobsCh) + }() - dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String()) + for i := 0; i < workers; i++ { + // Do jobs + go func() { + for job := range jobsCh { + chainBlock, err := localConnections.getChainBlock(job.BlockNumber) + if err != nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Unable to get block: %d from chain, err %v", job.BlockNumber, err), + ErrorSource: "blockchain", + Number: job.BlockNumber, + } + continue + } - if err != nil { - description := fmt.Sprintf("Unable to get block: %d, err: %v", bn, err) - fmt.Println(description) - corruptBlocks.registerCorruptBlock(bn, "database", description) - continue + dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String()) + + if err != nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Unable to get block: %d, err: %v", job.BlockNumber, err), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + if dbBlock.Number == nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Block %d not presented in database", job.BlockNumber), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + if chainBlock.NumberU64() != dbBlock.Number.Uint64() { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Incorrect %d block retrieved from database", job.BlockNumber), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + chainTxs := localConnections.getChainTxs(chainBlock.Hash(), job.BlockNumber) + + if len(chainTxs) != len(dbBlock.Transactions) { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Different number of transactions in block %d, err %v", job.BlockNumber, err), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + job.Results <- Result{ + Output: fmt.Sprintf("Processed block number: %d", job.BlockNumber), + } + } + doneCh <- struct{}{} + }() + } + + // Await completion + go func() { + for i := 0; i < workers; i++ { + <-doneCh } + close(resultCh) + }() - if dbBlock.Number == nil { - description := fmt.Sprintf("Block %d not presented in database", bn) - fmt.Println(description) - corruptBlocks.registerCorruptBlock(bn, "database", description) - continue + for result := range resultCh { + if result.ErrorOutput != "" { + fmt.Println(result.ErrorOutput) + corruptBlocks.registerCorruptBlock(result.Number, result.ErrorSource, result.ErrorOutput) } - - if chainBlock.NumberU64() != dbBlock.Number.Uint64() { - description := fmt.Sprintf("Incorrect %d block retrieved from database", bn) - fmt.Println(description) - corruptBlocks.registerCorruptBlock(bn, "database", description) - continue + if result.Output != "" { + fmt.Println(result.Output) } - - chainTxs := localConnections.getChainTxs(chainBlock.Hash(), bn) - - if len(chainTxs) != len(dbBlock.Transactions) { - description := fmt.Sprintf("Different number of transactions in block %d, err %v", bn, err) - fmt.Println(description) - corruptBlocks.registerCorruptBlock(bn, "database", description) - continue + if result.Output != "" && result.ErrorOutput != "" { + fmt.Printf("Unprocessable result with error: %s and output: %s", result.ErrorOutput, result.Output) } - - fmt.Printf("Processed block number: %d\r", bn) } return nil diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 8f43b267..4eae0478 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -8,7 +8,7 @@ import ( "sort" "strconv" - "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + // "github.com/bugout-dev/moonstream/crawlers/ldb/configs" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/google/uuid" @@ -33,9 +33,15 @@ var ( Usage: `Blockchain garbage collection mode ("full", "archive")`, Value: "full", } + ThreadsFlag = cli.IntFlag{ + Name: "threads", + Usage: "Number of threads to use", + Value: 2, + } ) // Block step generator, yield list of blocks with length equal blockStep +// TODO(kompotkot): Not-safe method with slices in channel, re-write this function func BlockYield(start, end, blockStep uint64) chan []uint64 { ch := make(chan []uint64) @@ -173,6 +179,10 @@ func processVerifyCommand(ctx *cli.Context) error { if blockchain != "ethereum" && blockchain != "polygon" { return fmt.Errorf("Unsupported blockchain provided") } + threads := ctx.GlobalInt(ThreadsFlag.Name) + if threads <= 0 { + threads = 1 + } start, end, err := startEndBlock(ctx) if err != nil { @@ -191,23 +201,11 @@ func processVerifyCommand(ctx *cli.Context) error { return fmt.Errorf("Unable to set database connection: %v", err) } - cnt := uint64(0) - reportStart := uint64(start) for blocks := range BlockYield(start, end, BlockNumberStep) { - err = verify(blockchain, blocks) + err = verify(blockchain, blocks, threads) if err != nil { return fmt.Errorf("Error occurred due verify acction: %v", err) } - - cnt += BlockNumberStep - if cnt >= configs.BLOCK_RANGE_REPORT { - err := humbugReporter.submitReport(reportStart, blocks[len(blocks)-1]+1, "") - if err != nil { - return fmt.Errorf("Unable to send humbug report: %v", err) - } - reportStart = blocks[len(blocks)-1] + 1 - cnt = 0 - } } err = humbugReporter.submitReport(start, end, "Total ") @@ -230,6 +228,7 @@ func LDBCLI() { BlockchainFlag, DataDirFlag, GCModeFlag, + ThreadsFlag, } app.Commands = []cli.Command{ @@ -267,6 +266,7 @@ func LDBCLI() { BlockchainFlag, DataDirFlag, GCModeFlag, + ThreadsFlag, }, }, } diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go index 19914539..3145f01f 100644 --- a/crawlers/ldb/configs/settings.go +++ b/crawlers/ldb/configs/settings.go @@ -5,8 +5,6 @@ import ( "time" ) -var BLOCK_RANGE_REPORT uint64 = 100000 - // Database configs var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute From d55209df4244f4b18ee2ae25d4ebe0eb66a05d0b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 23:21:24 +0000 Subject: [PATCH 34/55] Code structure organization --- crawlers/ldb/cmd/actions.go | 13 ------------- crawlers/ldb/cmd/data.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go index 5e462306..15ea5e4b 100644 --- a/crawlers/ldb/cmd/actions.go +++ b/crawlers/ldb/cmd/actions.go @@ -73,19 +73,6 @@ func show(blockNumbers []uint64) error { return nil } -// TODO(kompotkot): Find way to remove Number -type Result struct { - ErrorOutput string - ErrorSource string - Number uint64 - Output string -} - -type Job struct { - BlockNumber uint64 - Results chan<- Result -} - // Run verification flow of blockchain with database data func verify(blockchain string, blockNumbers []uint64, workers int) error { jobsCh := make(chan Job, workers) diff --git a/crawlers/ldb/cmd/data.go b/crawlers/ldb/cmd/data.go index 20e7cab8..ce816165 100644 --- a/crawlers/ldb/cmd/data.go +++ b/crawlers/ldb/cmd/data.go @@ -52,3 +52,17 @@ type CorruptBlock struct { type CorruptBlocks struct { Blocks []CorruptBlock `json:"blocks"` } + +// Concurrency jobs structure +type Job struct { + BlockNumber uint64 + Results chan<- Result +} + +// TODO(kompotkot): Find way to remove Number +type Result struct { + ErrorOutput string + ErrorSource string + Number uint64 + Output string +} From 1473ca3b4084abbf92cdc6878c626b55b6df98c2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Sun, 6 Mar 2022 23:33:28 +0000 Subject: [PATCH 35/55] Fixes --- crawlers/ldb/cmd/cli.go | 5 +---- crawlers/ldb/cmd/data.go | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go index 4eae0478..97e5e44b 100644 --- a/crawlers/ldb/cmd/cli.go +++ b/crawlers/ldb/cmd/cli.go @@ -8,8 +8,6 @@ import ( "sort" "strconv" - // "github.com/bugout-dev/moonstream/crawlers/ldb/configs" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/google/uuid" "gopkg.in/urfave/cli.v1" @@ -61,7 +59,7 @@ func BlockYield(start, end, blockStep uint64) chan []uint64 { var blocks []uint64 for i := currentBlock; i < tempEnd; i++ { - blocks = append(blocks, i) // Block operation + blocks = append(blocks, i) // Blocking operation } ch <- blocks @@ -75,7 +73,6 @@ func BlockYield(start, end, blockStep uint64) chan []uint64 { } // Parse start and end blocks from command line input -// TODO(kompotkot): Re-write to work via channel in goroutines func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { inputStart, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) if err != nil { diff --git a/crawlers/ldb/cmd/data.go b/crawlers/ldb/cmd/data.go index ce816165..4650d6be 100644 --- a/crawlers/ldb/cmd/data.go +++ b/crawlers/ldb/cmd/data.go @@ -59,7 +59,7 @@ type Job struct { Results chan<- Result } -// TODO(kompotkot): Find way to remove Number +// TODO(kompotkot): Find way to remove Number, it repeats Job type Result struct { ErrorOutput string ErrorSource string From 6da2dcacae30118e645a602302ff0cfc0544bb55 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 7 Mar 2022 12:24:30 +0000 Subject: [PATCH 36/55] Removed status systemd check to prevent deploy script freeze --- nodes/deploy/ethereum/deploy.bash | 1 - nodes/deploy/polygon/deploy.bash | 1 - 2 files changed, 2 deletions(-) diff --git a/nodes/deploy/ethereum/deploy.bash b/nodes/deploy/ethereum/deploy.bash index 03d8e17d..75d63649 100755 --- a/nodes/deploy/ethereum/deploy.bash +++ b/nodes/deploy/ethereum/deploy.bash @@ -65,7 +65,6 @@ chmod 644 "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODE_STATUS_SERVER_SERVICE_FILE}" systemctl daemon-reload systemctl restart "${NODE_STATUS_SERVER_SERVICE_FILE}" -systemctl status "${NODE_STATUS_SERVER_SERVICE_FILE}" echo echo diff --git a/nodes/deploy/polygon/deploy.bash b/nodes/deploy/polygon/deploy.bash index 17230994..39acfaa0 100755 --- a/nodes/deploy/polygon/deploy.bash +++ b/nodes/deploy/polygon/deploy.bash @@ -69,7 +69,6 @@ chmod 644 "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODE_STATUS_SERVER_SERVICE_FILE}" systemctl daemon-reload systemctl restart "${NODE_STATUS_SERVER_SERVICE_FILE}" -systemctl status "${NODE_STATUS_SERVER_SERVICE_FILE}" echo echo From 6d56b01dff6631b0b0f79bbc74aef8fee4ceffe1 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 7 Mar 2022 19:14:56 +0200 Subject: [PATCH 37/55] Add fixes. --- backend/moonstreamapi/data.py | 6 ++ backend/moonstreamapi/routes/queries.py | 119 +++++++++++++++--------- backend/moonstreamapi/settings.py | 14 ++- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/backend/moonstreamapi/data.py b/backend/moonstreamapi/data.py index 5c497330..76b6aec1 100644 --- a/backend/moonstreamapi/data.py +++ b/backend/moonstreamapi/data.py @@ -12,6 +12,8 @@ from sqlalchemy import false USER_ONBOARDING_STATE = "onboarding_state" +BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" + class TimeScale(Enum): month = "month" @@ -278,3 +280,7 @@ class PreapprovedQuery(BaseModel): query: str name: str public: bool = False + + +class QueryPresignUrl(BaseModel): + url: str diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 67936b37..23f7c617 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -23,19 +23,19 @@ from ..settings import ( MOONSTREAM_CRAWLERS_SERVER_PORT, MOONSTREAM_QUERIES_BUCKET, MOONSTREAM_QUERIES_JOURNAL_ID, - BUGOUT_RESOURCE_QUERY_RESOLVER, ) from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter( - prefix="/queries", -) +router = APIRouter(prefix="/queries",) -BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" +class ResourceQueryFetchException(Exception): + """ + Exception in queries API + """ @router.get("/list", tags=["queries"]) @@ -46,13 +46,13 @@ async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: # Check already existed queries params = { - "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, } try: resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error( f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" ) @@ -79,15 +79,15 @@ async def create_query_handler( # Check already existed queries params = { - "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, } try: resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error( - f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" + f"Error create query for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" ) raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -95,7 +95,11 @@ async def create_query_handler( resource.resource_data["name"] for resource in resources.resources ] - query_name = slugify(query_applied.name) + try: + query_name = slugify(query_applied.name) + except ResourceQueryFetchException as e: + logger.error(f"Error in query normalization.") + raise MoonstreamHTTPException(status_code=500, internal_error=e) if query_name in used_queries: @@ -115,7 +119,7 @@ async def create_query_handler( ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error creating query entry: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -125,7 +129,7 @@ async def create_query_handler( token=token, application_id=MOONSTREAM_APPLICATION_ID, resource_data={ - "type": BUGOUT_RESOURCE_QUERY_RESOLVER, + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "user_id": str(user.id), "user": str(user.username), "name": query_name, @@ -135,7 +139,7 @@ async def create_query_handler( except BugoutResponseException as e: logger.error(f"Error creating name resolving resource: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error creating name resolving resource: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -151,7 +155,7 @@ async def create_query_handler( except BugoutResponseException as e: logger.error(f"Error in applind tags to query entry: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error in applind tags to query entry: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -163,7 +167,11 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE token = request.state.token - query_id = get_query_by_name(query_name, token) + try: + query_id = get_query_by_name(query_name, token) + except ResourceQueryFetchException as e: + logger.error(f"Error in request query by name from brood resources: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -174,11 +182,11 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE ) except BugoutResponseException as e: - logger.error(f"Error in updating query: {str(e)}") + logger.error(f"Error in get query: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: - logger.error(f"Error in updating query: {e}") + except ResourceQueryFetchException as e: + logger.error(f"Error in get query: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry @@ -193,7 +201,11 @@ async def update_query_handler( token = request.state.token - query_id = get_query_by_name(query_name, token) + try: + query_id = get_query_by_name(query_name, token) + except ResourceQueryFetchException as e: + logger.error(f"Error in request query by name from brood resources: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -210,26 +222,34 @@ async def update_query_handler( logger.error(f"Error in updating query: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error in updating query: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry -@router.post("/{query_name}/update_data", tags=["queries"]) +@router.post( + "/{query_name}/update_data", + tags=["queries"], + response_model=Optional[data.QueryPresignUrl], +) async def update_query_data_handler( request: Request, query_name: str, request_update: data.UpdateDataRequest = Body(...), -) -> Optional[Dict[str, Any]]: +) -> Optional[data.QueryPresignUrl]: """ Request update data on S3 bucket """ token = request.state.token - query_id = get_query_by_name(query_name, token) + try: + query_id = get_query_by_name(query_name, token) + except ResourceQueryFetchException as e: + logger.error(f"Error in request query by name from brood resources: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) try: entries = bc.search( @@ -240,7 +260,14 @@ async def update_query_data_handler( timeout=5, ) - if entries.results and entries.results[0].content: + if len(entries.results) == 0: + raise MoonstreamHTTPException( + status_code=403, detail="Query not approved yet." + ) + + s3_response = None + + if entries.results[0].content: content = entries.results[0].content tags = entries.results[0].tags @@ -266,18 +293,19 @@ async def update_query_data_handler( detail="Task for start generate stats failed.", ) - return responce.json() - except Exception as e: + s3_response = data.QueryPresignUrl(**responce.json()) + except ResourceQueryFetchException as e: logger.error(f"Error in send generate query data task: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) - raise MoonstreamHTTPException(status_code=403, detail="Query not approved yet.") + return s3_response -@router.get("/{query_name}", tags=["queries"]) +@router.get( + "/{query_name}", tags=["queries"], response_model=Optional[data.QueryPresignUrl] +) async def get_access_link_handler( - request: Request, - query_name: str, -) -> str: + request: Request, query_name: str, +) -> Optional[data.QueryPresignUrl]: """ Request update data on S3 bucket """ @@ -286,7 +314,11 @@ async def get_access_link_handler( token = request.state.token - query_id = get_query_by_name(query_name, token) + try: + query_id = get_query_by_name(query_name, token) + except ResourceQueryFetchException as e: + logger.error(f"Error in request query by name from brood resources: {e}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) s3 = boto3.client("s3") @@ -294,11 +326,13 @@ async def get_access_link_handler( entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved #query_id:{query_id} !#preapprove", + query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", limit=1, timeout=5, ) + s3_response = None + if entries.results and entries.results[0].content: tags = entries.results[0].tags @@ -317,17 +351,16 @@ async def get_access_link_handler( ExpiresIn=300000, HttpMethod="GET", ) - return stats_presigned_url - except Exception as e: + s3_response = data.QueryPresignUrl(url=stats_presigned_url) + except ResourceQueryFetchException as e: logger.error(f"Error in send generate query data task: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) - raise MoonstreamHTTPException(status_code=403, detail="Query not approved yet.") + return s3_response @router.delete("/{query_name}", tags=["queries"]) async def remove_query_handler( - request: Request, - query_name: str, + request: Request, query_name: str, ) -> BugoutJournalEntry: """ Request update data on S3 bucket @@ -342,12 +375,12 @@ async def remove_query_handler( timeout: float = REQUESTS_TIMEOUT """ - params = {"type": BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} try: resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error get query, error: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -365,7 +398,7 @@ async def remove_query_handler( bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error get query, error: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -377,7 +410,7 @@ async def remove_query_handler( ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: + except ResourceQueryFetchException as e: logger.error(f"Error get query, error: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 88935592..07ea0349 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -7,8 +7,6 @@ BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") -BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" - bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 @@ -70,8 +68,8 @@ if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None: raise ValueError( "MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set" ) -MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = ( - MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/") +MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip( + "/" ) MOONSTREAM_CRAWLERS_SERVER_URL = os.environ.get("MOONSTREAM_CRAWLERS_SERVER_URL") @@ -105,3 +103,11 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") if MOONSTREAM_QUERIES_BUCKET == "": raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") + +MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_QUERIES_BUCKET_PREFIX", "" +) +if MOONSTREAM_QUERIES_BUCKET_PREFIX == "": + raise ValueError( + "MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set" + ) From e2290aa49aa788ee330d78c13ea5876f5a44ac49 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 7 Mar 2022 19:16:59 +0200 Subject: [PATCH 38/55] Add fixes. --- backend/moonstreamapi/routes/queries.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 23f7c617..ee03e790 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -22,6 +22,7 @@ from ..settings import ( MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_CRAWLERS_SERVER_PORT, MOONSTREAM_QUERIES_BUCKET, + MOONSTREAM_QUERIES_BUCKET_PREFIX, MOONSTREAM_QUERIES_JOURNAL_ID, ) from ..settings import bugout_client as bc @@ -346,7 +347,7 @@ async def get_access_link_handler( "get_object", Params={ "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"queries/{query_id}/data.{file_type}", + "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/{query_id}/data.{file_type}", }, ExpiresIn=300000, HttpMethod="GET", @@ -367,14 +368,6 @@ async def remove_query_handler( """ token = request.state.token - """ - def delete_resource( - self, - token: Union[str, uuid.UUID], - resource_id: Union[str, uuid.UUID], - timeout: float = REQUESTS_TIMEOUT - """ - params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} try: resources: BugoutResources = bc.list_resources(token=token, params=params) From eac4f812ee16d58a7c9164992616c85581795dfc Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 7 Mar 2022 19:22:00 +0200 Subject: [PATCH 39/55] Fix import and response model. --- backend/moonstreamapi/actions.py | 23 +++++------------------ backend/moonstreamapi/routes/queries.py | 8 ++------ 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 7bc5d606..25d4192d 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -36,7 +36,6 @@ from .settings import ( MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, MOONSTREAM_MOONWORM_TASKS_JOURNAL, - BUGOUT_RESOURCE_QUERY_RESOLVER, ) from .settings import bugout_client as bc @@ -241,11 +240,7 @@ def create_onboarding_resource( token: uuid.UUID, resource_data: Dict[str, Any] = { "type": data.USER_ONBOARDING_STATE, - "steps": { - "welcome": 0, - "subscriptions": 0, - "stream": 0, - }, + "steps": {"welcome": 0, "subscriptions": 0, "stream": 0,}, "is_complete": False, }, ) -> BugoutResource: @@ -297,9 +292,7 @@ def json_type(evm_type: str) -> type: def dashboards_abi_validation( - dashboard_subscription: data.DashboardMeta, - abi: Any, - s3_path: str, + dashboard_subscription: data.DashboardMeta, abi: Any, s3_path: str, ): """ @@ -408,9 +401,7 @@ def validate_abi_json(abi: Any) -> None: def upload_abi_to_s3( - resource: BugoutResource, - abi: str, - update: Dict[str, Any], + resource: BugoutResource, abi: str, update: Dict[str, Any], ) -> Dict[str, Any]: """ Uploading ABI to s3 bucket. Return object for updating resource. @@ -482,11 +473,7 @@ def get_all_entries_from_search( return results -def apply_moonworm_tasks( - subscription_type: str, - abi: Any, - address: str, -) -> None: +def apply_moonworm_tasks(subscription_type: str, abi: Any, address: str,) -> None: """ Get list of subscriptions loads abi and apply them as moonworm tasks if it not exist """ @@ -546,7 +533,7 @@ def apply_moonworm_tasks( def get_query_by_name(query_name: str, token: uuid.UUID) -> str: - params = {"type": BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} try: resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index ee03e790..722da16a 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -231,9 +231,7 @@ async def update_query_handler( @router.post( - "/{query_name}/update_data", - tags=["queries"], - response_model=Optional[data.QueryPresignUrl], + "/{query_name}/update_data", tags=["queries"], ) async def update_query_data_handler( request: Request, @@ -301,9 +299,7 @@ async def update_query_data_handler( return s3_response -@router.get( - "/{query_name}", tags=["queries"], response_model=Optional[data.QueryPresignUrl] -) +@router.get("/{query_name}", tags=["queries"]) async def get_access_link_handler( request: Request, query_name: str, ) -> Optional[data.QueryPresignUrl]: From 94d30be625d896bd3f7a9b8c06244cfc3ef78f69 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Mon, 7 Mar 2022 20:08:35 +0200 Subject: [PATCH 40/55] Add name_normalization function. --- backend/moonstreamapi/actions.py | 18 ++++++++++++++++++ backend/moonstreamapi/routes/queries.py | 8 ++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 25d4192d..b5871605 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -19,6 +19,7 @@ from bugout.exceptions import BugoutResponseException from ens.utils import is_valid_ens_name # type: ignore from eth_utils.address import is_address # type: ignore from moonstreamdb.models import EthereumLabel +from slugify import slugify # type: ignore from sqlalchemy import text from sqlalchemy.orm import Session from web3 import Web3 @@ -531,8 +532,25 @@ def apply_moonworm_tasks(subscription_type: str, abi: Any, address: str,) -> Non ) +def name_normalization(query_name: str) -> str: + """ + return correct url name + """ + try: + normalized_query_name = slugify( + query_name, max_length=50, regex_pattern=r"[A-Za-z0-9_-]" + ) + except Exception as e: + logger.error(f"Error in query normalization.") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return normalized_query_name + + def get_query_by_name(query_name: str, token: uuid.UUID) -> str: + query_name = name_normalization(query_name) + params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} try: resources: BugoutResources = bc.list_resources(token=token, params=params) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 722da16a..e8ce7826 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -14,7 +14,7 @@ from slugify import slugify # type: ignore from .. import data -from ..actions import get_query_by_name +from ..actions import get_query_by_name, name_normalization from ..middleware import MoonstreamHTTPException from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -96,11 +96,7 @@ async def create_query_handler( resource.resource_data["name"] for resource in resources.resources ] - try: - query_name = slugify(query_applied.name) - except ResourceQueryFetchException as e: - logger.error(f"Error in query normalization.") - raise MoonstreamHTTPException(status_code=500, internal_error=e) + query_name = name_normalization(query_applied.name) if query_name in used_queries: From f5b2550505a0ded7f5601745ad6071a46176193b Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 8 Mar 2022 15:18:27 +0200 Subject: [PATCH 41/55] Add fixes. --- backend/moonstreamapi/actions.py | 4 +- backend/moonstreamapi/routes/queries.py | 15 ++++---- backend/moonstreamapi/version.py | 2 +- crawlers/mooncrawl/mooncrawl/api.py | 38 +++++++++++++------ crawlers/mooncrawl/mooncrawl/settings.py | 9 +++++ .../mooncrawl/stats_worker/queries.py | 5 +-- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index b5871605..7f3b29b6 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -537,9 +537,7 @@ def name_normalization(query_name: str) -> str: return correct url name """ try: - normalized_query_name = slugify( - query_name, max_length=50, regex_pattern=r"[A-Za-z0-9_-]" - ) + normalized_query_name = slugify(query_name, max_length=50) except Exception as e: logger.error(f"Error in query normalization.") raise MoonstreamHTTPException(status_code=500, internal_error=e) diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index e8ce7826..7c01a7ff 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -67,7 +67,7 @@ async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: @router.post("/", tags=["queries"]) async def create_query_handler( - request: Request, query_name: str, query_applied: data.PreapprovedQuery = Body(...) + request: Request, query_applied: data.PreapprovedQuery = Body(...) ) -> BugoutJournalEntry: """ Create query in bugout journal @@ -250,7 +250,7 @@ async def update_query_data_handler( entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"#approved #query_id:{query_id} !#preapprove", + query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", limit=1, timeout=5, ) @@ -284,8 +284,7 @@ async def update_query_data_handler( if responce.status_code != 200: raise MoonstreamHTTPException( - status_code=responce.status_code, - detail="Task for start generate stats failed.", + status_code=responce.status_code, detail=responce.text, ) s3_response = data.QueryPresignUrl(**responce.json()) @@ -300,7 +299,7 @@ async def get_access_link_handler( request: Request, query_name: str, ) -> Optional[data.QueryPresignUrl]: """ - Request update data on S3 bucket + Request S3 presign url """ # get real connect to query_id @@ -339,7 +338,7 @@ async def get_access_link_handler( "get_object", Params={ "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/{query_id}/data.{file_type}", + "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", }, ExpiresIn=300000, HttpMethod="GET", @@ -356,7 +355,7 @@ async def remove_query_handler( request: Request, query_name: str, ) -> BugoutJournalEntry: """ - Request update data on S3 bucket + Request delete query from journal """ token = request.state.token @@ -377,7 +376,7 @@ async def remove_query_handler( for resource in resources.resources } if len(query_ids) == 0: - raise MoonstreamHTTPException(status_code=404, detail="Query does not existю") + raise MoonstreamHTTPException(status_code=404, detail="Query does not exists") try: bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) diff --git a/backend/moonstreamapi/version.py b/backend/moonstreamapi/version.py index 01ed664c..bf3545dc 100644 --- a/backend/moonstreamapi/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.1.2" +MOONSTREAMAPI_VERSION = "0.1.3" diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index f511259c..e321a927 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -1,6 +1,7 @@ """ The Mooncrawl HTTP API """ +from cgi import test from datetime import datetime, timedelta import logging from os import times @@ -23,6 +24,7 @@ from .settings import ( bugout_client as bc, BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, MOONSTREAM_QUERIES_BUCKET, + MOONSTREAM_QUERIES_BUCKET_PREFIX, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from .version import MOONCRAWL_VERSION @@ -82,17 +84,14 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, - background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, - resource_id=stats_update.dashboard_id, - timeout=10, + token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, ) # get all user subscriptions @@ -120,7 +119,9 @@ async def status_handler( ) except Exception as e: - logger.error(f"Unhandled status exception, error: {e}") + logger.error( + f"Unhandled /jobs/stats_update start background task exception, error: {e}" + ) raise MoonstreamHTTPException(status_code=500) presigned_urls_response: Dict[UUID, Any] = {} @@ -172,13 +173,28 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, - request: data.QueryDataUpdate, - background_tasks: BackgroundTasks, + query_id: str, request: data.QueryDataUpdate, background_tasks: BackgroundTasks, ) -> Dict[str, Any]: s3_client = boto3.client("s3") + expected_query_parameters = text(request.query)._bindparams.keys() + + # request.params validations + passed_params = { + key: value + for key, value in request.params.values() + if key in expected_query_parameters + } + + if len(passed_params) != len(expected_query_parameters): + logger.error( + f"Unmatched amount of applying query parameters: {passed_params}, query_id:{query_id}." + ) + raise MoonstreamHTTPException( + status_code=500, detail="Unmatched amount of applying query parameters" + ) + try: background_tasks.add_task( @@ -191,14 +207,14 @@ async def queries_data_update_handler( ) except Exception as e: - logger.error(f"Unhandled status exception, error: {e}") + logger.error(f"Unhandled query execute exception, error: {e}") raise MoonstreamHTTPException(status_code=500) stats_presigned_url = s3_client.generate_presigned_url( "get_object", Params={ "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"queries/{query_id}/data.{request.file_type}", + "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request.file_type}", }, ExpiresIn=43200, # 12 hours HttpMethod="GET", diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 66b01bdc..3e52f743 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -89,3 +89,12 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "": MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") if MOONSTREAM_QUERIES_BUCKET == "": raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") + + +MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_QUERIES_BUCKET_PREFIX", "" +) +if MOONSTREAM_QUERIES_BUCKET_PREFIX == "": + raise ValueError( + "MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set" + ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 57f87aea..2386f725 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -7,8 +7,7 @@ import csv import boto3 # type: ignore from moonstreamdb.db import yield_db_session_ctx -from sqlalchemy import text - +from ..settings import MOONSTREAM_QUERIES_BUCKET_PREFIX logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -72,6 +71,6 @@ def data_generate( push_statistics( s3=s3, data=data, - key=f"queries/{query_id}/data.{file_type}", + key=f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", bucket=bucket, ) From 3c590484598d293572d65e39e0731ef51d03ee47 Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 8 Mar 2022 16:00:39 +0200 Subject: [PATCH 42/55] Fix issue with items. --- crawlers/mooncrawl/mooncrawl/api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index e321a927..e0f7ac9e 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -173,17 +173,19 @@ async def status_handler( @app.post("/jobs/{query_id}/query_update", tags=["jobs"]) async def queries_data_update_handler( - query_id: str, request: data.QueryDataUpdate, background_tasks: BackgroundTasks, + query_id: str, + request_data: data.QueryDataUpdate, + background_tasks: BackgroundTasks, ) -> Dict[str, Any]: s3_client = boto3.client("s3") - expected_query_parameters = text(request.query)._bindparams.keys() + expected_query_parameters = text(request_data.query)._bindparams.keys() # request.params validations passed_params = { key: value - for key, value in request.params.values() + for key, value in request_data.params.items() if key in expected_query_parameters } @@ -201,9 +203,9 @@ async def queries_data_update_handler( queries.data_generate, bucket=MOONSTREAM_QUERIES_BUCKET, query_id=f"{query_id}", - file_type=request.file_type, - query=request.query, - params=request.params, + file_type=request_data.file_type, + query=request_data.query, + params=request_data.params, ) except Exception as e: @@ -214,7 +216,7 @@ async def queries_data_update_handler( "get_object", Params={ "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request.file_type}", + "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}", }, ExpiresIn=43200, # 12 hours HttpMethod="GET", From 542f411449321dccbab721b421623b3c8dc34d7c Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 8 Mar 2022 17:06:50 +0200 Subject: [PATCH 43/55] Add using read only user. --- backend/moonstreamapi/actions.py | 11 ++++-- backend/moonstreamapi/routes/queries.py | 34 ++++++++++++++++--- .../mooncrawl/stats_worker/queries.py | 6 ++-- db/moonstreamdb/db.py | 2 +- db/moonstreamdb/version.py | 2 +- 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 7f3b29b6..fb73978f 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -57,6 +57,12 @@ class StatusAPIException(Exception): """ +class NameNormalizationException(Exception): + """ + Raised on actions when slugify can't normalize name. + """ + + class LabelNames(Enum): ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" COINMARKETCAP_TOKEN = "coinmarketcap_token" @@ -539,8 +545,9 @@ def name_normalization(query_name: str) -> str: try: normalized_query_name = slugify(query_name, max_length=50) except Exception as e: - logger.error(f"Error in query normalization.") - raise MoonstreamHTTPException(status_code=500, internal_error=e) + logger.error(f"Error in query normalization. Error: {e}") + + raise NameNormalizationException(f"Can't normalize name:{query_name}") return normalized_query_name diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 7c01a7ff..661d399a 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -10,11 +10,10 @@ from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJourna from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Request import requests -from slugify import slugify # type: ignore from .. import data -from ..actions import get_query_by_name, name_normalization +from ..actions import get_query_by_name, name_normalization, NameNormalizationException from ..middleware import MoonstreamHTTPException from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -95,14 +94,19 @@ async def create_query_handler( used_queries: List[str] = [ resource.resource_data["name"] for resource in resources.resources ] - - query_name = name_normalization(query_applied.name) + try: + query_name = name_normalization(query_applied.name) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) if query_name in used_queries: raise MoonstreamHTTPException( status_code=404, - detail=f"Provided query name already use. Please remove it or use PUT /{query_name}", + detail=f"Provided query name already use. Please remove it or use PUT /{query_name} for update query", ) try: @@ -166,6 +170,11 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE try: query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) except ResourceQueryFetchException as e: logger.error(f"Error in request query by name from brood resources: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -200,6 +209,11 @@ async def update_query_handler( try: query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) except ResourceQueryFetchException as e: logger.error(f"Error in request query by name from brood resources: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -242,6 +256,11 @@ async def update_query_data_handler( try: query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) except ResourceQueryFetchException as e: logger.error(f"Error in request query by name from brood resources: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -308,6 +327,11 @@ async def get_access_link_handler( try: query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) except ResourceQueryFetchException as e: logger.error(f"Error in request query by name from brood resources: {e}") raise MoonstreamHTTPException(status_code=500, internal_error=e) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 2386f725..a537209d 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -5,8 +5,8 @@ from io import StringIO import csv -import boto3 # type: ignore -from moonstreamdb.db import yield_db_session_ctx +import boto3 +from moonstreamdb.db import yield_db_read_only_session_ctx from ..settings import MOONSTREAM_QUERIES_BUCKET_PREFIX logging.basicConfig(level=logging.INFO) @@ -38,7 +38,7 @@ def data_generate( """ s3 = boto3.client("s3") - with yield_db_session_ctx() as db_session: + with yield_db_read_only_session_ctx() as db_session: if file_type == "csv": csv_buffer = StringIO() diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index 62e17fe6..17b4157c 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -91,4 +91,4 @@ def yield_db_read_only_session() -> Session: session.close() -yield_db_read_only_session_ctx = contextmanager(yield_db_session) +yield_db_read_only_session_ctx = contextmanager(yield_db_read_only_session) diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index 7f3d3eaa..80ecc71f 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.2" +MOONSTREAMDB_VERSION = "0.2.3" From 2077ea2c81bdc67ab0c1efca1627a0b0ca04e55d Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Tue, 8 Mar 2022 17:10:06 +0200 Subject: [PATCH 44/55] Add black formating. --- backend/moonstreamapi/actions.py | 20 ++++++++++++++++---- backend/moonstreamapi/routes/queries.py | 16 +++++++++++----- backend/moonstreamapi/settings.py | 4 ++-- crawlers/mooncrawl/mooncrawl/api.py | 7 +++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index fb73978f..d41d42ce 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -247,7 +247,11 @@ def create_onboarding_resource( token: uuid.UUID, resource_data: Dict[str, Any] = { "type": data.USER_ONBOARDING_STATE, - "steps": {"welcome": 0, "subscriptions": 0, "stream": 0,}, + "steps": { + "welcome": 0, + "subscriptions": 0, + "stream": 0, + }, "is_complete": False, }, ) -> BugoutResource: @@ -299,7 +303,9 @@ def json_type(evm_type: str) -> type: def dashboards_abi_validation( - dashboard_subscription: data.DashboardMeta, abi: Any, s3_path: str, + dashboard_subscription: data.DashboardMeta, + abi: Any, + s3_path: str, ): """ @@ -408,7 +414,9 @@ def validate_abi_json(abi: Any) -> None: def upload_abi_to_s3( - resource: BugoutResource, abi: str, update: Dict[str, Any], + resource: BugoutResource, + abi: str, + update: Dict[str, Any], ) -> Dict[str, Any]: """ Uploading ABI to s3 bucket. Return object for updating resource. @@ -480,7 +488,11 @@ def get_all_entries_from_search( return results -def apply_moonworm_tasks(subscription_type: str, abi: Any, address: str,) -> None: +def apply_moonworm_tasks( + subscription_type: str, + abi: Any, + address: str, +) -> None: """ Get list of subscriptions loads abi and apply them as moonworm tasks if it not exist """ diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 661d399a..e1972b22 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -29,7 +29,9 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) -router = APIRouter(prefix="/queries",) +router = APIRouter( + prefix="/queries", +) class ResourceQueryFetchException(Exception): @@ -241,7 +243,8 @@ async def update_query_handler( @router.post( - "/{query_name}/update_data", tags=["queries"], + "/{query_name}/update_data", + tags=["queries"], ) async def update_query_data_handler( request: Request, @@ -303,7 +306,8 @@ async def update_query_data_handler( if responce.status_code != 200: raise MoonstreamHTTPException( - status_code=responce.status_code, detail=responce.text, + status_code=responce.status_code, + detail=responce.text, ) s3_response = data.QueryPresignUrl(**responce.json()) @@ -315,7 +319,8 @@ async def update_query_data_handler( @router.get("/{query_name}", tags=["queries"]) async def get_access_link_handler( - request: Request, query_name: str, + request: Request, + query_name: str, ) -> Optional[data.QueryPresignUrl]: """ Request S3 presign url @@ -376,7 +381,8 @@ async def get_access_link_handler( @router.delete("/{query_name}", tags=["queries"]) async def remove_query_handler( - request: Request, query_name: str, + request: Request, + query_name: str, ) -> BugoutJournalEntry: """ Request delete query from journal diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 07ea0349..f6596e74 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -68,8 +68,8 @@ if MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX is None: raise ValueError( "MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set" ) -MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip( - "/" +MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX = ( + MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/") ) MOONSTREAM_CRAWLERS_SERVER_URL = os.environ.get("MOONSTREAM_CRAWLERS_SERVER_URL") diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index e0f7ac9e..68ea8b34 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -84,14 +84,17 @@ async def now_handler() -> data.NowResponse: @app.post("/jobs/stats_update", tags=["jobs"]) async def status_handler( - stats_update: data.StatsUpdateRequest, background_tasks: BackgroundTasks, + stats_update: data.StatsUpdateRequest, + background_tasks: BackgroundTasks, ): """ Update dashboard endpoint create are tasks for update. """ dashboard_resource: BugoutResource = bc.get_resource( - token=stats_update.token, resource_id=stats_update.dashboard_id, timeout=10, + token=stats_update.token, + resource_id=stats_update.dashboard_id, + timeout=10, ) # get all user subscriptions From c5e17534d74de7addfb13879b2fbf54f2607c8f3 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 11:20:23 +0000 Subject: [PATCH 45/55] DB read only env var in sample and docker --- db/configs/docker_generate_env.bash | 1 + db/configs/sample.env | 1 + 2 files changed, 2 insertions(+) diff --git a/db/configs/docker_generate_env.bash b/db/configs/docker_generate_env.bash index 0ef9c16f..306a1d3b 100755 --- a/db/configs/docker_generate_env.bash +++ b/db/configs/docker_generate_env.bash @@ -41,6 +41,7 @@ sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" +sed -i "s|^MOONSTREAM_DB_URI_READ_ONLY=.*|MOONSTREAM_DB_URI_READ_ONLY=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" # Generate alembic config diff --git a/db/configs/sample.env b/db/configs/sample.env index c5409401..8cfb9a45 100644 --- a/db/configs/sample.env +++ b/db/configs/sample.env @@ -1,3 +1,4 @@ # Required environment variables to work with database CLI export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 From ce9584a89eeefbbfc2527a3a592b256d3a553073 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 11:40:34 +0000 Subject: [PATCH 46/55] Moonstreamdb with read only --- db/configs/docker_generate_env.bash | 1 + db/configs/sample.env | 1 + db/moonstreamdb/db.py | 31 +++++++++++++++++++++++++++++ db/moonstreamdb/version.py | 2 +- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/db/configs/docker_generate_env.bash b/db/configs/docker_generate_env.bash index 0ef9c16f..306a1d3b 100755 --- a/db/configs/docker_generate_env.bash +++ b/db/configs/docker_generate_env.bash @@ -41,6 +41,7 @@ sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" +sed -i "s|^MOONSTREAM_DB_URI_READ_ONLY=.*|MOONSTREAM_DB_URI_READ_ONLY=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" # Generate alembic config diff --git a/db/configs/sample.env b/db/configs/sample.env index c5409401..8cfb9a45 100644 --- a/db/configs/sample.env +++ b/db/configs/sample.env @@ -1,3 +1,4 @@ # Required environment variables to work with database CLI export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index bfe055e9..4e6c7383 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -11,6 +11,10 @@ MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") if MOONSTREAM_DB_URI is None: raise ValueError("MOONSTREAM_DB_URI environment variable must be set") +MOONSTREAM_DB_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_URI_READ_ONLY") +if MOONSTREAM_DB_URI_READ_ONLY is None: + raise ValueError("MOONSTREAM_DB_URI_READ_ONLY environment variable must be set") + MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0) try: if MOONSTREAM_POOL_SIZE_RAW is not None: @@ -60,3 +64,30 @@ def yield_db_session() -> Session: yield_db_session_ctx = contextmanager(yield_db_session) + +# Read only + +RO_engine = create_engine( + MOONSTREAM_DB_URI_READ_ONLY, + pool_size=MOONSTREAM_POOL_SIZE, + connect_args={ + "options": f"-c statement_timeout={MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS}" + }, +) +RO_SessionLocal = sessionmaker(bind=RO_engine) + + +def yield_db_read_only_session() -> Session: + """ + Yields a database connection (created using environment variables). + As per FastAPI docs: + https://fastapi.tiangolo.com/tutorial/sql-databases/#create-a-dependency + """ + session = RO_SessionLocal() + try: + yield session + finally: + session.close() + + +yield_db_read_only_session_ctx = contextmanager(yield_db_read_only_session) diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index 7f3d3eaa..80ecc71f 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.2" +MOONSTREAMDB_VERSION = "0.2.3" From 29e84431a6e1bfa24f90c8eb0f0f4d653bac5fe5 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 12:08:31 +0000 Subject: [PATCH 47/55] Bumped db version, extended sample.end, docker config and mypy fix --- backend/configs/docker_generate_env.bash | 1 + backend/configs/sample.env | 1 + backend/setup.py | 2 +- crawlers/mooncrawl/mooncrawl/stats_worker/queries.py | 8 ++++---- crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/backend/configs/docker_generate_env.bash b/backend/configs/docker_generate_env.bash index f3db3b53..314b3c33 100755 --- a/backend/configs/docker_generate_env.bash +++ b/backend/configs/docker_generate_env.bash @@ -42,5 +42,6 @@ sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^MOONSTREAM_DB_URI_READ_ONLY=.*|MOONSTREAM_DB_URI_READ_ONLY=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^BUGOUT_BROOD_URL=.*|BUGOUT_BROOD_URL=$BUGOUT_BROOD_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^BUGOUT_SPIRE_URL=.*|BUGOUT_SPIRE_URL=$BUGOUT_SPIRE_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" diff --git a/backend/configs/sample.env b/backend/configs/sample.env index 8a6b4cdf..142b94c4 100644 --- a/backend/configs/sample.env +++ b/backend/configs/sample.env @@ -1,5 +1,6 @@ # Required environment variables export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" diff --git a/backend/setup.py b/backend/setup.py index 70b3ed44..b84c7a78 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -15,7 +15,7 @@ setup( "boto3", "bugout>=0.1.19", "fastapi", - "moonstreamdb>=0.2.2", + "moonstreamdb>=0.2.3", "humbug", "pydantic", "pyevmasm", diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index a537209d..5e856892 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -1,12 +1,12 @@ +import csv import json import logging -from typing import Any, Dict, Optional from io import StringIO -import csv +from typing import Any, Dict, Optional - -import boto3 +import boto3 # type: ignore from moonstreamdb.db import yield_db_read_only_session_ctx + from ..settings import MOONSTREAM_QUERIES_BUCKET_PREFIX logging.basicConfig(level=logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 48874ac7..586fbde4 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.1.3" +MOONCRAWL_VERSION = "0.1.4" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 73e388f8..20e06983 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -5,6 +5,7 @@ export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://=0.1.19", "chardet", "fastapi", - "moonstreamdb>=0.2.2", + "moonstreamdb>=0.2.3", "moonworm==0.1.11", "humbug", "pydantic", From 57faaf814e0d55060d5c87112f08a022541ad427 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 12:29:02 +0000 Subject: [PATCH 48/55] Fixed exceptions --- backend/moonstreamapi/actions.py | 17 ++++-- backend/moonstreamapi/routes/queries.py | 69 +++++++++---------------- 2 files changed, 37 insertions(+), 49 deletions(-) diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index d41d42ce..1d6a52f0 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -62,6 +62,11 @@ class NameNormalizationException(Exception): Raised on actions when slugify can't normalize name. """ +class ResourceQueryFetchException(Exception): + """ + Exception in queries API + """ + class LabelNames(Enum): ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" @@ -552,21 +557,25 @@ def apply_moonworm_tasks( def name_normalization(query_name: str) -> str: """ - return correct url name + Sanitize provided query name. """ try: normalized_query_name = slugify(query_name, max_length=50) except Exception as e: logger.error(f"Error in query normalization. Error: {e}") - raise NameNormalizationException(f"Can't normalize name:{query_name}") return normalized_query_name def get_query_by_name(query_name: str, token: uuid.UUID) -> str: - - query_name = name_normalization(query_name) + """ + Fetch query_id from Brood resources. + """ + try: + query_name = name_normalization(query_name) + except Exception: + raise NameNormalizationException("Unable to normalize query name") params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} try: diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index e1972b22..83ba383b 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -33,13 +33,6 @@ router = APIRouter( prefix="/queries", ) - -class ResourceQueryFetchException(Exception): - """ - Exception in queries API - """ - - @router.get("/list", tags=["queries"]) async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: @@ -54,10 +47,7 @@ async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error( - f"Error listing subscriptions for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" - ) + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) users_queries: List[Dict[str, Any]] = [ @@ -87,10 +77,7 @@ async def create_query_handler( resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error( - f"Error create query for user ({request.user.id}) with token ({request.state.token}), error: {str(e)}" - ) + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) used_queries: List[str] = [ @@ -122,8 +109,7 @@ async def create_query_handler( ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error creating query entry: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -142,8 +128,7 @@ async def create_query_handler( except BugoutResponseException as e: logger.error(f"Error creating name resolving resource: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error creating name resolving resource: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -158,8 +143,7 @@ async def create_query_handler( except BugoutResponseException as e: logger.error(f"Error in applind tags to query entry: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error in applind tags to query entry: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry @@ -177,8 +161,7 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE status_code=403, detail=f"Provided query name can't be normalize please select different.", ) - except ResourceQueryFetchException as e: - logger.error(f"Error in request query by name from brood resources: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -192,9 +175,7 @@ async def get_query_handler(request: Request, query_name: str) -> BugoutJournalE except BugoutResponseException as e: logger.error(f"Error in get query: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - - except ResourceQueryFetchException as e: - logger.error(f"Error in get query: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry @@ -216,8 +197,7 @@ async def update_query_handler( status_code=403, detail=f"Provided query name can't be normalize please select different.", ) - except ResourceQueryFetchException as e: - logger.error(f"Error in request query by name from brood resources: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -234,9 +214,7 @@ async def update_query_handler( except BugoutResponseException as e: logger.error(f"Error in updating query: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - - except ResourceQueryFetchException as e: - logger.error(f"Error in updating query: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry @@ -264,8 +242,7 @@ async def update_query_data_handler( status_code=403, detail=f"Provided query name can't be normalize please select different.", ) - except ResourceQueryFetchException as e: - logger.error(f"Error in request query by name from brood resources: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -311,9 +288,12 @@ async def update_query_data_handler( ) s3_response = data.QueryPresignUrl(**responce.json()) - except ResourceQueryFetchException as e: - logger.error(f"Error in send generate query data task: {e}") + except BugoutResponseException as e: + logger.error(f"Error in updating query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) + return s3_response @@ -337,8 +317,7 @@ async def get_access_link_handler( status_code=403, detail=f"Provided query name can't be normalize please select different.", ) - except ResourceQueryFetchException as e: - logger.error(f"Error in request query by name from brood resources: {e}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) s3 = boto3.client("s3") @@ -373,9 +352,12 @@ async def get_access_link_handler( HttpMethod="GET", ) s3_response = data.QueryPresignUrl(url=stats_presigned_url) - except ResourceQueryFetchException as e: - logger.error(f"Error in send generate query data task: {e}") + except BugoutResponseException as e: + logger.error(f"Error in get access link: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) + return s3_response @@ -394,8 +376,7 @@ async def remove_query_handler( resources: BugoutResources = bc.list_resources(token=token, params=params) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error get query, error: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) query_ids: Dict[str, Tuple[UUID, Union[UUID, str]]] = { @@ -412,8 +393,7 @@ async def remove_query_handler( bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error get query, error: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) try: @@ -424,8 +404,7 @@ async def remove_query_handler( ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except ResourceQueryFetchException as e: - logger.error(f"Error get query, error: {str(e)}") + except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) return entry From b408c20b0bce29390be421b6a5115dce7e4c881f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 12:44:54 +0000 Subject: [PATCH 49/55] Added missed env variables in sample.env --- backend/configs/sample.env | 7 +++++-- backend/moonstreamapi/routes/queries.py | 8 +++---- backend/moonstreamapi/settings.py | 14 ++++++------- crawlers/mooncrawl/mooncrawl/api.py | 10 ++++----- crawlers/mooncrawl/mooncrawl/settings.py | 14 ++++++------- .../mooncrawl/stats_worker/queries.py | 4 ++-- crawlers/mooncrawl/sample.env | 21 ++++++++++--------- 7 files changed, 41 insertions(+), 37 deletions(-) diff --git a/backend/configs/sample.env b/backend/configs/sample.env index 142b94c4..7da28466 100644 --- a/backend/configs/sample.env +++ b/backend/configs/sample.env @@ -8,18 +8,21 @@ export MOONSTREAM_APPLICATION_ID="" export MOONSTREAM_ADMIN_ACCESS_TOKEN="" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_MOONWORM_TASKS_JOURNAL="" +export MOONSTREAM_CRAWLERS_SERVER_URL="" +export MOONSTREAM_CRAWLERS_SERVER_PORT="" # Blockchain, txpool, whalewatch data depends variables export MOONSTREAM_DATA_JOURNAL_ID="" export HUMBUG_TXPOOL_CLIENT_ID="" export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_QUERIES_JOURNAL_ID="" - +export MOONSTREAM_QUERIES_JOURNAL_ID="" # Set following parameters if AWS node instance and S3 smartcontracts configured export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" +export MOONSTREAM_S3_QUERIES_BUCKET="" +export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev" # Set the following variables in the most reasonable manner for your development environment export HUMBUG_REPORTER_BACKEND_TOKEN="" diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 83ba383b..0cbc85e2 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -20,8 +20,8 @@ from ..settings import ( MOONSTREAM_APPLICATION_ID, MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_CRAWLERS_SERVER_PORT, - MOONSTREAM_QUERIES_BUCKET, - MOONSTREAM_QUERIES_BUCKET_PREFIX, + MOONSTREAM_S3_QUERIES_BUCKET, + MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, MOONSTREAM_QUERIES_JOURNAL_ID, ) from ..settings import bugout_client as bc @@ -345,8 +345,8 @@ async def get_access_link_handler( stats_presigned_url = s3.generate_presigned_url( "get_object", Params={ - "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", + "Bucket": MOONSTREAM_S3_QUERIES_BUCKET, + "Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", }, ExpiresIn=300000, HttpMethod="GET", diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index f6596e74..ad858797 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -100,14 +100,14 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable must be set" ) -MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") -if MOONSTREAM_QUERIES_BUCKET == "": - raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") +MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "") +if MOONSTREAM_S3_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set") -MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get( - "MOONSTREAM_QUERIES_BUCKET_PREFIX", "" +MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", "" ) -if MOONSTREAM_QUERIES_BUCKET_PREFIX == "": +if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "": raise ValueError( - "MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set" + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set" ) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 68ea8b34..03275849 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -23,8 +23,8 @@ from .settings import ( ORIGINS, bugout_client as bc, BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, - MOONSTREAM_QUERIES_BUCKET, - MOONSTREAM_QUERIES_BUCKET_PREFIX, + MOONSTREAM_S3_QUERIES_BUCKET, + MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from .version import MOONCRAWL_VERSION @@ -204,7 +204,7 @@ async def queries_data_update_handler( background_tasks.add_task( queries.data_generate, - bucket=MOONSTREAM_QUERIES_BUCKET, + bucket=MOONSTREAM_S3_QUERIES_BUCKET, query_id=f"{query_id}", file_type=request_data.file_type, query=request_data.query, @@ -218,8 +218,8 @@ async def queries_data_update_handler( stats_presigned_url = s3_client.generate_presigned_url( "get_object", Params={ - "Bucket": MOONSTREAM_QUERIES_BUCKET, - "Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}", + "Bucket": MOONSTREAM_S3_QUERIES_BUCKET, + "Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}", }, ExpiresIn=43200, # 12 hours HttpMethod="GET", diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 3e52f743..13494c55 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -86,15 +86,15 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "": ) -MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "") -if MOONSTREAM_QUERIES_BUCKET == "": - raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set") +MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "") +if MOONSTREAM_S3_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set") -MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get( - "MOONSTREAM_QUERIES_BUCKET_PREFIX", "" +MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", "" ) -if MOONSTREAM_QUERIES_BUCKET_PREFIX == "": +if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "": raise ValueError( - "MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set" + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set" ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 5e856892..9311a274 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional import boto3 # type: ignore from moonstreamdb.db import yield_db_read_only_session_ctx -from ..settings import MOONSTREAM_QUERIES_BUCKET_PREFIX +from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -71,6 +71,6 @@ def data_generate( push_statistics( s3=s3, data=data, - key=f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", + key=f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", bucket=bucket, ) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 20e06983..57871f7e 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -6,14 +6,15 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https:// Date: Wed, 9 Mar 2022 13:05:31 +0000 Subject: [PATCH 50/55] Test cases for name_normalization --- backend/moonstreamapi/actions.py | 5 ++++- backend/moonstreamapi/routes/queries.py | 1 + backend/moonstreamapi/text_actions.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 backend/moonstreamapi/text_actions.py diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 1d6a52f0..eb774327 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -62,6 +62,7 @@ class NameNormalizationException(Exception): Raised on actions when slugify can't normalize name. """ + class ResourceQueryFetchException(Exception): """ Exception in queries API @@ -560,7 +561,9 @@ def name_normalization(query_name: str) -> str: Sanitize provided query name. """ try: - normalized_query_name = slugify(query_name, max_length=50) + normalized_query_name = slugify( + query_name, max_length=50, lowercase=False, separator="_" + ) except Exception as e: logger.error(f"Error in query normalization. Error: {e}") raise NameNormalizationException(f"Can't normalize name:{query_name}") diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py index 0cbc85e2..18b2718f 100644 --- a/backend/moonstreamapi/routes/queries.py +++ b/backend/moonstreamapi/routes/queries.py @@ -33,6 +33,7 @@ router = APIRouter( prefix="/queries", ) + @router.get("/list", tags=["queries"]) async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: diff --git a/backend/moonstreamapi/text_actions.py b/backend/moonstreamapi/text_actions.py new file mode 100644 index 00000000..17fb3a2d --- /dev/null +++ b/backend/moonstreamapi/text_actions.py @@ -0,0 +1,19 @@ +import unittest + +from . import actions + + +class TestActions(unittest.TestCase): + def test_name_normalization(self): + names = [ + ["test", "test"], + ["test_Name", "test_Name"], + ["%20UNION", "20UNION"], + ["UNION ALL", "UNION_ALL"], + ["$_REQUEST", "REQUEST"], + ["id=1", "id_1"], + ["Lo" * 30, "Lo" * 25], + ] + for name in names: + query_name = actions.name_normalization(name[0]) + self.assertEqual(query_name, name[1]) From d3754386683c5def57a29dad12de884cbf543f3b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 9 Mar 2022 15:51:49 +0000 Subject: [PATCH 51/55] Query validator with tests --- crawlers/mooncrawl/mooncrawl/api.py | 13 +++- .../mooncrawl/stats_worker/queries.py | 19 ++++++ .../mooncrawl/stats_worker/test_queries.py | 65 +++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 03275849..9724ff01 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -200,6 +200,17 @@ async def queries_data_update_handler( status_code=500, detail="Unmatched amount of applying query parameters" ) + try: + valid_query = queries.query_validation(request_data.query) + except queries.QueryNotValid: + logger.error(f"Incorrect query provided with id: {query_id}") + raise MoonstreamHTTPException( + status_code=401, detail="Incorrect query provided" + ) + except Exception as e: + logger.error(f"Unhandled query execute exception, error: {e}") + raise MoonstreamHTTPException(status_code=500) + try: background_tasks.add_task( @@ -207,7 +218,7 @@ async def queries_data_update_handler( bucket=MOONSTREAM_S3_QUERIES_BUCKET, query_id=f"{query_id}", file_type=request_data.file_type, - query=request_data.query, + query=valid_query, params=request_data.params, ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 9311a274..86a8b419 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -1,6 +1,7 @@ import csv import json import logging +import re from io import StringIO from typing import Any, Dict, Optional @@ -12,6 +13,14 @@ from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +QUERY_REGEX = re.compile("[\[\]@#$%^&?;`/]") + + +class QueryNotValid(Exception): + """ + Raised when query validation not passed. + """ + def push_statistics(s3: Any, data: Any, key: str, bucket: str) -> None: @@ -26,6 +35,16 @@ def push_statistics(s3: Any, data: Any, key: str, bucket: str) -> None: logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") +def query_validation(query: str) -> str: + """ + Sanitize provided query. + """ + if QUERY_REGEX.search(query) != None: + raise QueryNotValid("Query contains restricted symbols") + + return query + + def data_generate( bucket: str, query_id: str, diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py new file mode 100644 index 00000000..a71911d7 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py @@ -0,0 +1,65 @@ +import unittest + +from . import queries + + +class TestQueries(unittest.TestCase): + def test_query_validation(self): + q = "SELECT * FROM ethereum_blocks" + self.assertEqual(queries.query_validation(q), q) + + q = "select count(*), tx_dublicates from ( select count(*) as tx_dublicates from polygon_labels where address = '0x123' and label_data->>'name' = 'Transfer' group by transaction_hash, log_index order by tx_dublicates desc) as dublicates group by dublicates.tx_dublicates" + self.assertEqual(queries.query_validation(q), q) + + q = """ + Select difference.address, + difference.transfers_in - difference.transfers_out as count + from ( + SELECT total.address, + sum(total.transfer_out) as transfers_out, + sum(total.transfer_in) as transfers_in + from ( + SELECT label_data->'args'->>'from' as address, + (label_data->'args'->'values'->>0)::int as transfer_out, + 0 as transfer_in + from polygon_labels + where address = '0x123' + and label = 'moonworm' + and label_data->>'name' = 'TransferBatch' + and (label_data->'args'->'ids'->>0)::int in (2, 3, 4) + UNION ALL + SELECT label_data->'args'->>'from' as address, + (label_data->'args'->'value')::int as transfer_out, + 0 as transfer_in + from polygon_labels + where address = '0x123' + and label = 'moonworm' + and label_data->>'name' = 'TransferSingle' + and (label_data->'args'->>'id')::int in (2, 3, 4) + ) as total + group by address + ) difference + order by count desc + """ + self.assertEqual(queries.query_validation(q), q) + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("SELECT hash FROM ethereum_transaction;") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("%20UNION") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("?id=1") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("FROM`") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("WHERE login='[USER]'") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("OR(1=1)#") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("/etc/hosts") From 792dfd55d6c9130f254ead3ba432f08474bded80 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 15 Mar 2022 17:15:50 +0000 Subject: [PATCH 52/55] update landing page --- frontend/pages/index.js | 41 ++++++++---- frontend/src/components/TrustedBadge.js | 85 +++++++++++++++---------- 2 files changed, 80 insertions(+), 46 deletions(-) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index b5d2c269..098219e0 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -19,6 +19,8 @@ import { GridItem, SimpleGrid, Button, + LinkBox, + LinkOverlay, Image as ChakraImage, } from "@chakra-ui/react"; import dynamic from "next/dynamic"; @@ -128,6 +130,7 @@ const assets = { cryptoTraders: `${AWS_ASSETS_PATH}/crypto+traders.png`, smartDevelopers: `${AWS_ASSETS_PATH}/smart+contract+developers.png`, cointelegraph: `${AWS_ASSETS_PATH}/featured_by/Cointelegraph_logo.png`, + forte: `${AWS_ASSETS_PATH}/featured_by/forte_logo.png`, educativesessions: `${AWS_ASSETS_PATH}/featured_by/educative_logo.png`, cryptoinsiders: `${AWS_ASSETS_PATH}/featured_by/crypto_insiders.png`, cryptoslate: `${AWS_ASSETS_PATH}/featured_by/cs-media-logo-light.png`, @@ -135,7 +138,7 @@ const assets = { ethereumBlackLogo: `${AWS_ASSETS_PATH}/eth-diamond-black.png`, ethereumRainbowLogo: `${AWS_ASSETS_PATH}/eth-diamond-rainbow.png`, maticLogo: `${AWS_ASSETS_PATH}/matic-token-inverted-icon.png`, - erc20: `${AWS_ASSETS_PATH}/ERC 20.png`, + lender: `${AWS_ASSETS_PATH}/lender.png`, DAO: `${AWS_ASSETS_PATH}/DAO .png`, NFT: `${AWS_ASSETS_PATH}/NFT.png`, bc101: `${AWS_ASSETS_PATH}/featured_by/blockchain101_logo.png`, @@ -503,17 +506,17 @@ const Homepage = () => { - Bridges + Engine - Open up your economy to Ethereum’s rich DeFi ecosystem, - no matter which chain you live on. + Create and manage tokens with custom mechanics specific + to your project. @@ -603,17 +606,26 @@ const Homepage = () => { name="Laguna games" caseURL="" ImgURL={assets["laguna"]} + boxURL="https://laguna.games/" /> + @@ -739,29 +751,28 @@ const Homepage = () => { }} elementName={"element3"} colorScheme="orange" - badge={`Moonstream bridges`} + badge={`Moonstream engine`} bullets={[ { - text: `Make it easy for participants to bring value into your economy from other blockchains.`, + text: `Deploy customizable and upgradable characters, items, and currencies into your economy`, icon: GiCrossedChains, color: "orange.50", bgColor: "orange.900", }, { - text: `Make your tokens and NFTs multi-chain.`, + text: `Monitor interactions between these tokens`, icon: GiChainedHeart, color: "orange.50", bgColor: "orange.900", }, { - text: `Run the cost-intensive parts of your economy off of Ethereum while tapping into Ethereum’s DeFi ecosystem. - `, + text: `Secure the tokens with Moonstream defense bots.`, icon: GiChaingun, color: "orange.50", bgColor: "orange.900", }, ]} - imgURL={assets["erc20"]} + imgURL={assets["lender"]} /> { name="cointelegraph" caseURL="" ImgURL={assets["cointelegraph"]} + boxURL="https://cointelegraph.com/news/17-of-addresses-snapped-up-80-of-all-ethereum-nfts-since-april" /> diff --git a/frontend/src/components/TrustedBadge.js b/frontend/src/components/TrustedBadge.js index f6dec843..1ce738b7 100644 --- a/frontend/src/components/TrustedBadge.js +++ b/frontend/src/components/TrustedBadge.js @@ -1,43 +1,60 @@ import { React } from "react"; -import { Flex, Image, Link } from "@chakra-ui/react"; +import { Flex, Image, Link, LinkBox, LinkOverlay } from "@chakra-ui/react"; -const TrustedBadge = ({ name, caseURL, ImgURL, scale, isGrayScale }) => { +const TrustedBadge = ({ + name, + caseURL, + ImgURL, + scale, + isGrayScale, + boxURL, +}) => { const _scale = scale ?? 1; return ( - - {name} - {caseURL && ( - // - + - {`Read more >`} - - // - )} - + {name} + {caseURL && ( + // + + {`Read more >`} + + // + )} + + + ); }; export default TrustedBadge; From dd57dfb4ffb931245e5dbe1dc7706571fb2510f4 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 15 Mar 2022 17:24:14 +0000 Subject: [PATCH 53/55] team page update --- frontend/pages/team/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/pages/team/index.js b/frontend/pages/team/index.js index 508e680d..73ca5348 100644 --- a/frontend/pages/team/index.js +++ b/frontend/pages/team/index.js @@ -248,8 +248,8 @@ const Product = () => { avatarURL={assets["dragonfly"]} name={"Tim Pechersky"} atName={"@peersky"} - content={`Spectral hopper. Hes special ability to precieve world trough spectral domain. Occasionaly - shifts in to time domain to spend some in doing flow + content={`Spectral hopper. Has special ability to perceive world trough spectral domain. Occasionaly + shifts in time domain to spend some in doing flow arts, surfing, and connecting with nature.`} /> Date: Tue, 15 Mar 2022 17:25:22 +0000 Subject: [PATCH 54/55] happy linter - happy life --- frontend/pages/index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index 098219e0..acb9214c 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -19,8 +19,6 @@ import { GridItem, SimpleGrid, Button, - LinkBox, - LinkOverlay, Image as ChakraImage, } from "@chakra-ui/react"; import dynamic from "next/dynamic"; From 56b82c9a19debacc7894bda62df479ef85a21c67 Mon Sep 17 00:00:00 2001 From: Tim Pechersky Date: Tue, 15 Mar 2022 21:42:31 +0000 Subject: [PATCH 55/55] improvements --- frontend/pages/index.js | 4 ++-- frontend/src/components/TrustedBadge.js | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/frontend/pages/index.js b/frontend/pages/index.js index acb9214c..ba3c0f76 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -617,7 +617,7 @@ const Homepage = () => { scale={1.5} name="orangedao" ImgURL={assets["orangedao"]} - boxURL="https://game7.io/" + boxURL="https://lfg.orangedao.xyz/" /> { scale={1.5} name="bc101" ImgURL={assets["bc101"]} - boxURL="https://blockchain101.com/courses/" + boxURL="https://blockchain101.com/" /> { const _scale = scale ?? 1; return ( - - + +