diff --git a/.gitignore b/.gitignore index 9ad5297a..c26689a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .secrets/ +.idea/ .vscode/ -.DS_Store \ No newline at end of file +.DS_Store diff --git a/backend/configs/docker_generate_env.bash b/backend/configs/docker_generate_env.bash index f3db3b53..314b3c33 100755 --- a/backend/configs/docker_generate_env.bash +++ b/backend/configs/docker_generate_env.bash @@ -42,5 +42,6 @@ sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" +sed -i "s|^MOONSTREAM_DB_URI_READ_ONLY=.*|MOONSTREAM_DB_URI_READ_ONLY=$DOCKER_MOONSTREAM_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^BUGOUT_BROOD_URL=.*|BUGOUT_BROOD_URL=$BUGOUT_BROOD_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" sed -i "s|^BUGOUT_SPIRE_URL=.*|BUGOUT_SPIRE_URL=$BUGOUT_SPIRE_URL|" "$SCRIPT_DIR/$DOCKER_MOONSTREAM_ENV_FILE" diff --git a/backend/configs/sample.env b/backend/configs/sample.env index 07bcb5d6..7da28466 100644 --- a/backend/configs/sample.env +++ b/backend/configs/sample.env @@ -1,5 +1,6 @@ # Required environment variables export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" export BUGOUT_BROOD_URL="https://auth.bugout.dev" export BUGOUT_SPIRE_URL="https://spire.bugout.dev" @@ -7,16 +8,21 @@ export MOONSTREAM_APPLICATION_ID="" export MOONSTREAM_ADMIN_ACCESS_TOKEN="" export MOONSTREAM_POOL_SIZE=0 export MOONSTREAM_MOONWORM_TASKS_JOURNAL="" +export MOONSTREAM_CRAWLERS_SERVER_URL="" +export MOONSTREAM_CRAWLERS_SERVER_PORT="" # Blockchain, txpool, whalewatch data depends variables export MOONSTREAM_DATA_JOURNAL_ID="" export HUMBUG_TXPOOL_CLIENT_ID="" export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_QUERIES_JOURNAL_ID="" # Set following parameters if AWS node instance and S3 smartcontracts configured export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="" export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="" +export MOONSTREAM_S3_QUERIES_BUCKET="" +export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev" # Set the following variables in the most reasonable manner for your development environment export HUMBUG_REPORTER_BACKEND_TOKEN="" diff --git a/backend/moonstreamapi/actions.py b/backend/moonstreamapi/actions.py index 8468b00b..eb774327 100644 --- a/backend/moonstreamapi/actions.py +++ b/backend/moonstreamapi/actions.py @@ -8,11 +8,18 @@ import uuid import boto3 # type: ignore -from bugout.data import BugoutSearchResults, BugoutSearchResult, BugoutResource +from bugout.data import ( + BugoutSearchResults, + BugoutSearchResult, + BugoutResource, + BugoutResources, +) from bugout.journal import SearchOrder +from bugout.exceptions import BugoutResponseException from ens.utils import is_valid_ens_name # type: ignore from eth_utils.address import is_address # type: ignore from moonstreamdb.models import EthereumLabel +from slugify import slugify # type: ignore from sqlalchemy import text from sqlalchemy.orm import Session from web3 import Web3 @@ -50,6 +57,18 @@ class StatusAPIException(Exception): """ +class NameNormalizationException(Exception): + """ + Raised on actions when slugify can't normalize name. + """ + + +class ResourceQueryFetchException(Exception): + """ + Exception in queries API + """ + + class LabelNames(Enum): ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" COINMARKETCAP_TOKEN = "coinmarketcap_token" @@ -535,3 +554,49 @@ def apply_moonworm_tasks( entries=entries_pack, timeout=15, ) + + +def name_normalization(query_name: str) -> str: + """ + Sanitize provided query name. + """ + try: + normalized_query_name = slugify( + query_name, max_length=50, lowercase=False, separator="_" + ) + except Exception as e: + logger.error(f"Error in query normalization. Error: {e}") + raise NameNormalizationException(f"Can't normalize name:{query_name}") + + return normalized_query_name + + +def get_query_by_name(query_name: str, token: uuid.UUID) -> str: + """ + Fetch query_id from Brood resources. + """ + try: + query_name = name_normalization(query_name) + except Exception: + raise NameNormalizationException("Unable to normalize query name") + + params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error get query, error: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + available_queries: Dict[str, str] = { + resource.resource_data["name"]: resource.resource_data["entry_id"] + for resource in resources.resources + } + + if query_name not in available_queries: + raise MoonstreamHTTPException(status_code=404, detail="Query not found.") + + query_id = available_queries[query_name] + + return query_id diff --git a/backend/moonstreamapi/api.py b/backend/moonstreamapi/api.py index 2f7db4f8..0ba1fcee 100644 --- a/backend/moonstreamapi/api.py +++ b/backend/moonstreamapi/api.py @@ -12,6 +12,7 @@ from . import actions, data from .middleware import BroodAuthMiddleware, MoonstreamHTTPException from .routes.address_info import router as addressinfo_router from .routes.dashboards import router as dashboards_router +from .routes.queries import router as queries_router from .routes.streams import router as streams_router from .routes.subscriptions import router as subscriptions_router from .routes.txinfo import router as txinfo_router @@ -30,6 +31,8 @@ tags_metadata = [ "name": "labels", "description": "Labels for transactions, addresses with additional information.", }, + {"name": "dashboards", "description": "Operations with user dashboards."}, + {"name": "queries", "description": "Operations with user queries."}, {"name": "streams", "description": "Operations with data streams and filters."}, {"name": "subscriptions", "description": "Operations with user subscriptions."}, {"name": "time", "description": "Server timestamp endpoints."}, @@ -127,3 +130,4 @@ app.include_router(txinfo_router) app.include_router(users_router) app.include_router(whales_router) app.include_router(dashboards_router) +app.include_router(queries_router) diff --git a/backend/moonstreamapi/data.py b/backend/moonstreamapi/data.py index f333d618..76b6aec1 100644 --- a/backend/moonstreamapi/data.py +++ b/backend/moonstreamapi/data.py @@ -3,13 +3,17 @@ Pydantic schemas for the Moonstream HTTP API """ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Literal from uuid import UUID +from xmlrpc.client import Boolean from pydantic import BaseModel, Field +from sqlalchemy import false USER_ONBOARDING_STATE = "onboarding_state" +BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" + class TimeScale(Enum): month = "month" @@ -262,3 +266,21 @@ class DashboardCreate(BaseModel): class DashboardUpdate(BaseModel): name: Optional[str] subscription_settings: List[DashboardMeta] = Field(default_factory=list) + + +class UpdateDataRequest(BaseModel): + params: Dict[str, Any] = Field(default_factory=dict) + + +class UpdateQueryRequest(BaseModel): + query: str + + +class PreapprovedQuery(BaseModel): + query: str + name: str + public: bool = False + + +class QueryPresignUrl(BaseModel): + url: str diff --git a/backend/moonstreamapi/routes/queries.py b/backend/moonstreamapi/routes/queries.py new file mode 100644 index 00000000..18b2718f --- /dev/null +++ b/backend/moonstreamapi/routes/queries.py @@ -0,0 +1,411 @@ +""" +The Moonstream queries HTTP API +""" +import logging +from typing import Any, Dict, List, Optional, Tuple, Union +from uuid import UUID + +import boto3 # type: ignore +from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry +from bugout.exceptions import BugoutResponseException +from fastapi import APIRouter, Body, Request +import requests + + +from .. import data +from ..actions import get_query_by_name, name_normalization, NameNormalizationException +from ..middleware import MoonstreamHTTPException +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_APPLICATION_ID, + MOONSTREAM_CRAWLERS_SERVER_URL, + MOONSTREAM_CRAWLERS_SERVER_PORT, + MOONSTREAM_S3_QUERIES_BUCKET, + MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, + MOONSTREAM_QUERIES_JOURNAL_ID, +) +from ..settings import bugout_client as bc + + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/queries", +) + + +@router.get("/list", tags=["queries"]) +async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: + + token = request.state.token + + # Check already existed queries + + params = { + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, + } + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + users_queries: List[Dict[str, Any]] = [ + resource.resource_data for resource in resources.resources + ] + return users_queries + + +@router.post("/", tags=["queries"]) +async def create_query_handler( + request: Request, query_applied: data.PreapprovedQuery = Body(...) +) -> BugoutJournalEntry: + """ + Create query in bugout journal + """ + + token = request.state.token + + user = request.state.user + + # Check already existed queries + + params = { + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, + } + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + used_queries: List[str] = [ + resource.resource_data["name"] for resource in resources.resources + ] + try: + query_name = name_normalization(query_applied.name) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) + + if query_name in used_queries: + + raise MoonstreamHTTPException( + status_code=404, + detail=f"Provided query name already use. Please remove it or use PUT /{query_name} for update query", + ) + + try: + # Put query to journal + entry = bc.create_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + title=f"Query:{query_name}", + tags=["type:query"], + content=query_applied.query, + ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + # create resource query_name_resolver + bc.create_resource( + token=token, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data={ + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, + "user_id": str(user.id), + "user": str(user.username), + "name": query_name, + "entry_id": str(entry.id), + }, + ) + except BugoutResponseException as e: + logger.error(f"Error creating name resolving resource: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + + bc.update_tags( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=entry.id, + tags=[f"query_id:{entry.id}", f"preapprove"], + ) + + except BugoutResponseException as e: + logger.error(f"Error in applind tags to query entry: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + + +@router.get("/{query_name}/query", tags=["queries"]) +async def get_query_handler(request: Request, query_name: str) -> BugoutJournalEntry: + + token = request.state.token + + try: + query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + + entry = bc.get_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_id, + ) + + except BugoutResponseException as e: + logger.error(f"Error in get query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + + +@router.put("/{query_name}", tags=["queries"]) +async def update_query_handler( + request: Request, + query_name: str, + request_update: data.UpdateQueryRequest = Body(...), +) -> BugoutJournalEntryContent: + + token = request.state.token + + try: + query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + + entry = bc.update_entry_content( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_id, + title=query_name, + content=request_update.query, + tags=["preapprove"], + ) + + except BugoutResponseException as e: + logger.error(f"Error in updating query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry + + +@router.post( + "/{query_name}/update_data", + tags=["queries"], +) +async def update_query_data_handler( + request: Request, + query_name: str, + request_update: data.UpdateDataRequest = Body(...), +) -> Optional[data.QueryPresignUrl]: + """ + Request update data on S3 bucket + """ + + token = request.state.token + + try: + query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + entries = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", + limit=1, + timeout=5, + ) + + if len(entries.results) == 0: + raise MoonstreamHTTPException( + status_code=403, detail="Query not approved yet." + ) + + s3_response = None + + if entries.results[0].content: + content = entries.results[0].content + + tags = entries.results[0].tags + + file_type = "json" + + if "ext:csv" in tags: + file_type = "csv" + + responce = requests.post( + f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/{query_id}/query_update", + json={ + "query": content, + "params": request_update.params, + "file_type": file_type, + }, + timeout=5, + ) + + if responce.status_code != 200: + raise MoonstreamHTTPException( + status_code=responce.status_code, + detail=responce.text, + ) + + s3_response = data.QueryPresignUrl(**responce.json()) + except BugoutResponseException as e: + logger.error(f"Error in updating query: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return s3_response + + +@router.get("/{query_name}", tags=["queries"]) +async def get_access_link_handler( + request: Request, + query_name: str, +) -> Optional[data.QueryPresignUrl]: + """ + Request S3 presign url + """ + + # get real connect to query_id + + token = request.state.token + + try: + query_id = get_query_by_name(query_name, token) + except NameNormalizationException: + raise MoonstreamHTTPException( + status_code=403, + detail=f"Provided query name can't be normalize please select different.", + ) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + s3 = boto3.client("s3") + + try: + entries = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", + limit=1, + timeout=5, + ) + + s3_response = None + + if entries.results and entries.results[0].content: + + tags = entries.results[0].tags + + file_type = "json" + + if "ext:csv" in tags: + file_type = "csv" + + stats_presigned_url = s3.generate_presigned_url( + "get_object", + Params={ + "Bucket": MOONSTREAM_S3_QUERIES_BUCKET, + "Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", + }, + ExpiresIn=300000, + HttpMethod="GET", + ) + s3_response = data.QueryPresignUrl(url=stats_presigned_url) + except BugoutResponseException as e: + logger.error(f"Error in get access link: {str(e)}") + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return s3_response + + +@router.delete("/{query_name}", tags=["queries"]) +async def remove_query_handler( + request: Request, + query_name: str, +) -> BugoutJournalEntry: + """ + Request delete query from journal + """ + token = request.state.token + + params = {"type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, "name": query_name} + try: + resources: BugoutResources = bc.list_resources(token=token, params=params) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + query_ids: Dict[str, Tuple[UUID, Union[UUID, str]]] = { + resource.resource_data["name"]: ( + resource.id, + resource.resource_data["entry_id"], + ) + for resource in resources.resources + } + if len(query_ids) == 0: + raise MoonstreamHTTPException(status_code=404, detail="Query does not exists") + + try: + bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + entry = bc.delete_entry( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, + entry_id=query_ids[query_name][1], + ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return entry diff --git a/backend/moonstreamapi/settings.py b/backend/moonstreamapi/settings.py index 2882644a..ad858797 100644 --- a/backend/moonstreamapi/settings.py +++ b/backend/moonstreamapi/settings.py @@ -22,6 +22,12 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "") if MOONSTREAM_DATA_JOURNAL_ID == "": raise ValueError("MOONSTREAM_DATA_JOURNAL_ID environment variable must be set") + +MOONSTREAM_QUERIES_JOURNAL_ID = os.environ.get("MOONSTREAM_QUERIES_JOURNAL_ID", "") +if MOONSTREAM_DATA_JOURNAL_ID == "": + raise ValueError("MOONSTREAM_QUERIES_JOURNAL_ID environment variable must be set") + + MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") if MOONSTREAM_ADMIN_ACCESS_TOKEN == "": raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set") @@ -93,3 +99,15 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": raise ValueError( "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable must be set" ) + +MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "") +if MOONSTREAM_S3_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set") + +MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", "" +) +if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "": + raise ValueError( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set" + ) diff --git a/backend/moonstreamapi/text_actions.py b/backend/moonstreamapi/text_actions.py new file mode 100644 index 00000000..17fb3a2d --- /dev/null +++ b/backend/moonstreamapi/text_actions.py @@ -0,0 +1,19 @@ +import unittest + +from . import actions + + +class TestActions(unittest.TestCase): + def test_name_normalization(self): + names = [ + ["test", "test"], + ["test_Name", "test_Name"], + ["%20UNION", "20UNION"], + ["UNION ALL", "UNION_ALL"], + ["$_REQUEST", "REQUEST"], + ["id=1", "id_1"], + ["Lo" * 30, "Lo" * 25], + ] + for name in names: + query_name = actions.name_normalization(name[0]) + self.assertEqual(query_name, name[1]) diff --git a/backend/moonstreamapi/version.py b/backend/moonstreamapi/version.py index 01ed664c..bf3545dc 100644 --- a/backend/moonstreamapi/version.py +++ b/backend/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.1.2" +MOONSTREAMAPI_VERSION = "0.1.3" diff --git a/backend/setup.py b/backend/setup.py index 705a2b3c..b84c7a78 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -15,12 +15,13 @@ setup( "boto3", "bugout>=0.1.19", "fastapi", - "moonstreamdb>=0.2.2", + "moonstreamdb>=0.2.3", "humbug", "pydantic", "pyevmasm", "python-dateutil", "python-multipart", + "python-slugify", "uvicorn", "web3", ], diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go new file mode 100644 index 00000000..15ea5e4b --- /dev/null +++ b/crawlers/ldb/cmd/actions.go @@ -0,0 +1,177 @@ +package cmd + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + _ "github.com/lib/pq" +) + +var ( + // Block which not found in database or have inconsistencies with blockchain + corruptBlocks CorruptBlocks +) + +// Write down inconsistent state between database and blockchain +/* +- number (uint64): Block number +- source (string): Source of nonconformity [blockchain, database] +- description (string): Description of error, why block marked as malformed +*/ +func (cb *CorruptBlocks) registerCorruptBlock(number uint64, source, description string) { + cb.Blocks = append(cb.Blocks, CorruptBlock{ + Number: number, + Source: source, + Description: description, + }) +} + +// Add new blocks with transactions to database +func add(blockchain string, blockNumbers []uint64) error { + for _, bn := range blockNumbers { + block, err := localConnections.getChainBlock(bn) + if err != nil { + description := fmt.Sprintf("Unable to get block: %d from chain, err %v", bn, err) + fmt.Println(description) + corruptBlocks.registerCorruptBlock(bn, "blockchain", description) + continue + } + td := localConnections.Chain.GetTd(block.Hash(), block.NumberU64()) + + chainTxs := localConnections.getChainTxs(block.Hash(), bn) + + err = localConnections.writeDatabaseBlockTxs(blockchain, block, chainTxs, td) + if err != nil { + fmt.Printf("Error occurred due saving block %d with transactions in database: %v", bn, err) + } + + fmt.Printf("Processed block number: %d\r", bn) + } + + return nil +} + +// Return range of block hashes with transaction hashes from blockchain +func show(blockNumbers []uint64) error { + for _, bn := range blockNumbers { + block, err := localConnections.getChainBlock(bn) + if err != nil { + fmt.Printf("Unable to get block: %d from chain, err %v\n", bn, err) + continue + } + + chainTxs := localConnections.getChainTxs(block.Hash(), bn) + + var txs []common.Hash + for _, tx := range chainTxs { + txs = append(txs, tx.Hash()) + } + + fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs) + } + + return nil +} + +// Run verification flow of blockchain with database data +func verify(blockchain string, blockNumbers []uint64, workers int) error { + jobsCh := make(chan Job, workers) + resultCh := make(chan Result, len(blockNumbers)) + doneCh := make(chan struct{}, workers) + + // Add jobs + go func() { + for _, bn := range blockNumbers { + jobsCh <- Job{ + BlockNumber: bn, + Results: resultCh, + } + } + close(jobsCh) + }() + + for i := 0; i < workers; i++ { + // Do jobs + go func() { + for job := range jobsCh { + chainBlock, err := localConnections.getChainBlock(job.BlockNumber) + if err != nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Unable to get block: %d from chain, err %v", job.BlockNumber, err), + ErrorSource: "blockchain", + Number: job.BlockNumber, + } + continue + } + + dbBlock, err := localConnections.getDatabaseBlockTxs(blockchain, chainBlock.Hash().String()) + + if err != nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Unable to get block: %d, err: %v", job.BlockNumber, err), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + if dbBlock.Number == nil { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Block %d not presented in database", job.BlockNumber), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + if chainBlock.NumberU64() != dbBlock.Number.Uint64() { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Incorrect %d block retrieved from database", job.BlockNumber), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + chainTxs := localConnections.getChainTxs(chainBlock.Hash(), job.BlockNumber) + + if len(chainTxs) != len(dbBlock.Transactions) { + job.Results <- Result{ + ErrorOutput: fmt.Sprintf("Different number of transactions in block %d, err %v", job.BlockNumber, err), + ErrorSource: "database", + Number: job.BlockNumber, + } + continue + } + + job.Results <- Result{ + Output: fmt.Sprintf("Processed block number: %d", job.BlockNumber), + } + } + doneCh <- struct{}{} + }() + } + + // Await completion + go func() { + for i := 0; i < workers; i++ { + <-doneCh + } + close(resultCh) + }() + + for result := range resultCh { + if result.ErrorOutput != "" { + fmt.Println(result.ErrorOutput) + corruptBlocks.registerCorruptBlock(result.Number, result.ErrorSource, result.ErrorOutput) + } + if result.Output != "" { + fmt.Println(result.Output) + } + if result.Output != "" && result.ErrorOutput != "" { + fmt.Printf("Unprocessable result with error: %s and output: %s", result.ErrorOutput, result.Output) + } + } + + return nil +} diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go new file mode 100644 index 00000000..97e5e44b --- /dev/null +++ b/crawlers/ldb/cmd/cli.go @@ -0,0 +1,289 @@ +package cmd + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strconv" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/google/uuid" + "gopkg.in/urfave/cli.v1" +) + +var ( + // Block steps used to prevent long executor tasks and data loss possibility + BlockNumberStep = uint64(1000) + + BlockchainFlag = cli.StringFlag{ + Name: "blockchain", + Usage: `Which blockchain to crawl ("ethereum", "polygon")`, + } + DataDirFlag = cli.StringFlag{ + Name: "datadir", + Usage: "Data directory for the databases and keystore", + Value: "/home/ubuntu/nodes/ethereum", + } + GCModeFlag = cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode ("full", "archive")`, + Value: "full", + } + ThreadsFlag = cli.IntFlag{ + Name: "threads", + Usage: "Number of threads to use", + Value: 2, + } +) + +// Block step generator, yield list of blocks with length equal blockStep +// TODO(kompotkot): Not-safe method with slices in channel, re-write this function +func BlockYield(start, end, blockStep uint64) chan []uint64 { + ch := make(chan []uint64) + + go func(ch chan []uint64) { + currentBlock := start + var tempEnd uint64 + + for { + if currentBlock >= end { + break + } + + tempEnd = currentBlock + blockStep + if tempEnd >= end { + tempEnd = end + } + + var blocks []uint64 + for i := currentBlock; i < tempEnd; i++ { + blocks = append(blocks, i) // Blocking operation + } + ch <- blocks + + currentBlock += blockStep + } + + close(ch) + }(ch) + + return ch +} + +// Parse start and end blocks from command line input +func startEndBlock(ctx *cli.Context) (uint64, uint64, error) { + inputStart, err := strconv.ParseUint(ctx.Args().Get(0), 10, 32) + if err != nil { + return 0, 0, err + } + inputEnd, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) + if err != nil { + return 0, 0, err + } + + var start, end uint64 + if inputStart < inputEnd { + start = inputStart + end = inputEnd + } else if inputStart > inputEnd { + start = inputEnd + end = inputStart + } else { + return 0, 0, fmt.Errorf("Start and end block can't be equal") + } + + return start, end, nil +} + +func processAddCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + err = setDatabase() + if err != nil { + return fmt.Errorf("Unable to set database connection: %v", err) + } + + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = add(blockchain, blocks) + if err != nil { + return fmt.Errorf("Error occurred due add acction: %v", err) + } + } + + localConnections.Chain.Stop() + + return nil +} + +func processShowCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = show(blocks) + if err != nil { + return fmt.Errorf("Error occurred due show acction: %v", err) + } + } + + localConnections.Chain.Stop() + + return nil +} + +func processVerifyCommand(ctx *cli.Context) error { + if ctx.NArg() != 2 { + return fmt.Errorf("Required arguments: %v", ctx.Command.ArgsUsage) + } + blockchain := ctx.GlobalString(BlockchainFlag.Name) + if blockchain != "ethereum" && blockchain != "polygon" { + return fmt.Errorf("Unsupported blockchain provided") + } + threads := ctx.GlobalInt(ThreadsFlag.Name) + if threads <= 0 { + threads = 1 + } + + start, end, err := startEndBlock(ctx) + if err != nil { + return fmt.Errorf("Unable to parse block range: %v", err) + } + + err = setLocalChain(ctx) + if err != nil { + return fmt.Errorf("Unable to set blockchain: %v", err) + } + defer localConnections.Stack.Close() + defer localConnections.ChainDB.Close() + + err = setDatabase() + if err != nil { + return fmt.Errorf("Unable to set database connection: %v", err) + } + + for blocks := range BlockYield(start, end, BlockNumberStep) { + err = verify(blockchain, blocks, threads) + if err != nil { + return fmt.Errorf("Error occurred due verify acction: %v", err) + } + } + + err = humbugReporter.submitReport(start, end, "Total ") + if err != nil { + return fmt.Errorf("Unable to send humbug report: %v", err) + } + + localConnections.Chain.Stop() + + return nil +} + +func LDBCLI() { + app := cli.NewApp() + app.Name = filepath.Base(os.Args[0]) + app.Author = "Bugout.dev" + app.Email = "engineering@bugout.dev" + app.Usage = "blockchain ldb extractor command line interface" + app.Flags = []cli.Flag{ + BlockchainFlag, + DataDirFlag, + GCModeFlag, + ThreadsFlag, + } + + app.Commands = []cli.Command{ + { + Name: "add", + Action: utils.MigrateFlags(processAddCommand), + ArgsUsage: " ", + Usage: "Add new blocks with transactions to database", + Description: "This command request blocks from blockchain and adds to database.", + Flags: []cli.Flag{ + BlockchainFlag, + DataDirFlag, + GCModeFlag, + }, + }, + { + Name: "show", + Action: utils.MigrateFlags(processShowCommand), + ArgsUsage: " ", + Usage: "Show block with transactions", + Description: "This command print out requested blocks.", + Flags: []cli.Flag{ + BlockchainFlag, + DataDirFlag, + GCModeFlag, + }, + }, + { + Name: "verify", + Action: utils.MigrateFlags(processVerifyCommand), + ArgsUsage: " ", + Usage: "Verify blocks with transactions at database", + Description: "This command compare blocks in database and in blockchain for difference.", + Flags: []cli.Flag{ + BlockchainFlag, + DataDirFlag, + GCModeFlag, + ThreadsFlag, + }, + }, + } + + sort.Sort(cli.FlagsByName(app.Flags)) + sort.Sort(cli.CommandsByName(app.Commands)) + + // Initialize local connections + localConnections = &LocalConnections{} + + // Initialize humbug client to be able write data in Bugout journal + humbugReporter = &HumbugReporter{} + sessionID := uuid.New().String() + err := setHumbugClient(sessionID) + if err != nil { + log.Fatal(err) + } + + err = app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} diff --git a/crawlers/ldb/cmd/connections.go b/crawlers/ldb/cmd/connections.go new file mode 100644 index 00000000..ab34a542 --- /dev/null +++ b/crawlers/ldb/cmd/connections.go @@ -0,0 +1,247 @@ +package cmd + +import ( + "database/sql" + "fmt" + "math/big" + + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" + _ "github.com/lib/pq" + "gopkg.in/urfave/cli.v1" +) + +var ( + localConnections *LocalConnections +) + +// Modified lightweight go-ethereum function +// Source: github.com/ethereum/go-ethereum/cmd/geth/config.go +func defaultNodeConfig() node.Config { + cfg := node.DefaultConfig + cfg.Name = "geth" + return cfg +} + +// Establish connection with blockchain +func setLocalChain(ctx *cli.Context) error { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + } + + // Apply flags + utils.SetNodeConfig(ctx, &cfg.Node) + stack, err := node.New(&cfg.Node) + if err != nil { + return fmt.Errorf("Failed to create the protocol stack: %v", err) + } + localConnections.Stack = stack + + utils.SetEthConfig(ctx, stack, &cfg.Eth) + + chain, chainDB := utils.MakeChain(ctx, stack) + localConnections.Chain = chain + localConnections.ChainDB = chainDB + + return nil +} + +// Establish connection with database +func setDatabase() error { + db, err := sql.Open("postgres", configs.MOONSTREAM_DB_URI) + if err != nil { + return fmt.Errorf("DSN parse error or another database initialization error: %v", err) + } + + // Set the maximum number of concurrently idle connections, + // by default sql.DB allows a maximum of 2 idle connections. + db.SetMaxIdleConns(configs.MOONSTREAM_DB_MAX_IDLE_CONNS) + + // Set the maximum lifetime of a connection. + // Longer lifetime increase memory usage. + db.SetConnMaxLifetime(configs.MOONSTREAM_DB_CONN_MAX_LIFETIME) + + localConnections.Database = db + + return nil +} + +// Retrive block from blockchain +func (lc *LocalConnections) getChainBlock(number uint64) (*types.Block, error) { + block := lc.Chain.GetBlockByNumber(number) + if block == nil { + return nil, fmt.Errorf("Not found %d block in chain", number) + } + return block, nil +} + +// Retrive block transactions from blockchain +func (lc *LocalConnections) getChainTxs(blockHash common.Hash, number uint64) []*types.Transaction { + var transactions []*types.Transaction + body := rawdb.ReadBody(lc.ChainDB, blockHash, number) + for _, tx := range body.Transactions { + transactions = append(transactions, tx) + } + + return transactions +} + +// Retrive block with transactions from database +func (lc *LocalConnections) getDatabaseBlockTxs(blockchain, blockHash string) (LightBlock, error) { + var lBlock LightBlock + var txs []LightTransaction + query := fmt.Sprintf( + `SELECT + %s_blocks.hash, + %s_blocks.block_number, + %s_transactions.hash + FROM %s_blocks + LEFT JOIN %s_transactions ON %s_blocks.block_number = %s_transactions.block_number + WHERE %s_blocks.hash = '%s';`, + blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockchain, blockHash, + ) + rows, err := lc.Database.Query(query) + if err != nil { + return lBlock, fmt.Errorf("An error occurred during sql operation: %v", err) + } + defer rows.Close() + + for rows.Next() { + var blockHash, blockNumberStr, txHash sql.NullString + err := rows.Scan(&blockHash, &blockNumberStr, &txHash) + if err != nil { + return lBlock, fmt.Errorf("An error occurred during sql operation: %v", err) + } + + var lTx LightTransaction + if txHash.Valid != false { + lTx = LightTransaction{ + Hash: txHash.String, + } + txs = append(lBlock.Transactions, lTx) + } + + blockNumber := new(big.Int) + blockNumber, ok := blockNumber.SetString(blockNumberStr.String, 10) + if !ok { + return lBlock, fmt.Errorf("Unable to parse block number") + } + + lBlock = LightBlock{ + Hash: blockHash.String, + Number: blockNumber, + Transactions: txs, + } + } + + return lBlock, nil +} + +// Write block with transactions to database +func (lc *LocalConnections) writeDatabaseBlockTxs( + blockchain string, block *types.Block, txs []*types.Transaction, td *big.Int, +) error { + // block.extraData doesn't exist at Polygon mainnet + var extraData interface{} + if block.Extra() == nil { + extraData = "NULL" + } else { + extraData = fmt.Sprintf("'0x%x'", block.Extra()) + } + + // For support of blocks before London hardfork + var baseFee interface{} + if block.BaseFee() == nil { + baseFee = "NULL" + } else { + baseFee = block.BaseFee() + } + + blockQuery := fmt.Sprintf( + `INSERT INTO %s_blocks ( + block_number, + difficulty, + extra_data, + gas_limit, + gas_used, + base_fee_per_gas, + hash, + logs_bloom, + miner, + nonce, + parent_hash, + receipt_root, + uncles, + size, + state_root, + timestamp, + total_difficulty, + transactions_root + ) VALUES (%d, %d, %v, %d, %d, %v, '%s', '0x%x', '%s', '0x%x', '0x%x', '0x%x', '0x%x', %f, '0x%x', %d, %d, '0x%x');`, + blockchain, + block.Number(), + block.Difficulty(), + extraData, + block.GasLimit(), + block.GasUsed(), + baseFee, + block.Hash(), + block.Bloom(), + block.Coinbase(), + block.Nonce(), + block.ParentHash(), + block.ReceiptHash(), + block.UncleHash(), + block.Size(), + block.Root(), + block.Time(), + td, + block.TxHash(), + ) + _, err := lc.Database.Exec(blockQuery) + if err != nil { + return fmt.Errorf("An error occurred during sql operation: %v", err) + } + + // for _, tx := range(txs) { + // txQuery := fmt.Sprintf( + // `INSERT INTO %s_transactions ( + // hash, + // block_number, + // from_address, + // to_address, + // gas, + // gas_price, + // max_fee_per_gas, + // max_priority_fee_per_gas, + // input, + // nonce, + // transaction_index, + // transaction_type, + // value + // ) VALUES ('%s', %d, '%s', '%s', %d, %d, );`, + // blockchain, + // tx.Hash(), + // block.Number(), + // tx.from, + // tx.To(), + // tx.Gas(), + // tx.GasPrice(), + // "max_fee", + // "max_prior", + // tx.input, + // tx.Nonce(), + // "tx_index", + // tx.Type(), + // tx.Value(), + // ) + // } + return nil +} diff --git a/crawlers/ldb/cmd/data.go b/crawlers/ldb/cmd/data.go new file mode 100644 index 00000000..4650d6be --- /dev/null +++ b/crawlers/ldb/cmd/data.go @@ -0,0 +1,68 @@ +package cmd + +import ( + "database/sql" + "math/big" + + humbug "github.com/bugout-dev/humbug/go/pkg" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/node" +) + +// Modified lightweight go-ethereum struct +// Source: github.com/ethereum/go-ethereum/cmd/geth/config.go +type gethConfig struct { + Eth ethconfig.Config + Node node.Config +} + +// Predefined connections to blockchain and database +type LocalConnections struct { + Stack *node.Node + Chain *core.BlockChain + ChainDB ethdb.Database + Database *sql.DB +} + +type HumbugReporter struct { + Reporter *humbug.HumbugReporter +} + +// Lightweight transactions for database operations +type LightTransaction struct { + Hash string +} + +// Lightweight block for database operations +type LightBlock struct { + Hash string + Number *big.Int + Transactions []LightTransaction +} + +// Malformed block structure which will be submitted to humbug journal +type CorruptBlock struct { + Number uint64 `json:"number"` + Source string `json:"source"` + Description string `json:"description"` +} + +type CorruptBlocks struct { + Blocks []CorruptBlock `json:"blocks"` +} + +// Concurrency jobs structure +type Job struct { + BlockNumber uint64 + Results chan<- Result +} + +// TODO(kompotkot): Find way to remove Number, it repeats Job +type Result struct { + ErrorOutput string + ErrorSource string + Number uint64 + Output string +} diff --git a/crawlers/ldb/cmd/reporter.go b/crawlers/ldb/cmd/reporter.go new file mode 100644 index 00000000..bd4e2cd2 --- /dev/null +++ b/crawlers/ldb/cmd/reporter.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "encoding/json" + "fmt" + + "github.com/bugout-dev/humbug/go/pkg" + "github.com/bugout-dev/moonstream/crawlers/ldb/configs" +) + +var ( + humbugReporter *HumbugReporter +) + +// Generate humbug client +func setHumbugClient(sessionID string) error { + consent := humbug.CreateHumbugConsent(humbug.True) + reporter, err := humbug.CreateHumbugReporter(consent, configs.HUMBUG_LDB_CLIENT_ID, sessionID, configs.HUMBUG_LDB_TOKEN) + if err != nil { + return fmt.Errorf("Unable to generate humbug reporter: %v", err) + } + humbugReporter.Reporter = reporter + + return nil +} + +// Publish report with verified blocks to entry +func (r *HumbugReporter) submitReport(start, end uint64, prefix string) error { + content, err := json.Marshal(corruptBlocks) + if err != nil { + return fmt.Errorf("Unable to marshal to json: %v", err) + } + + report := humbug.Report{ + Title: fmt.Sprintf("%sLDB verifier %d-%d", prefix, start, end), + Content: string(content), + Tags: []string{ + fmt.Sprintf("start:%d", start), + fmt.Sprintf("end:%d", end), + }, + } + r.Reporter.Publish(report) + fmt.Printf("%sLDB report for range %d-%d published\n", prefix, start, end) + + return nil +} diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go new file mode 100644 index 00000000..3145f01f --- /dev/null +++ b/crawlers/ldb/configs/settings.go @@ -0,0 +1,15 @@ +package configs + +import ( + "os" + "time" +) + +// Database configs +var MOONSTREAM_DB_MAX_IDLE_CONNS int = 30 +var MOONSTREAM_DB_CONN_MAX_LIFETIME = 30 * time.Minute +var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI") + +// Humber configs +var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID") +var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN") diff --git a/crawlers/ldb/go.mod b/crawlers/ldb/go.mod new file mode 100644 index 00000000..ac48656a --- /dev/null +++ b/crawlers/ldb/go.mod @@ -0,0 +1,64 @@ +module github.com/bugout-dev/moonstream/crawlers/ldb + +go 1.17 + +require ( + github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 + github.com/ethereum/go-ethereum v1.10.16 + github.com/google/uuid v1.1.5 + github.com/lib/pq v1.0.0 + gopkg.in/urfave/cli.v1 v1.20.0 +) + +require ( + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v1.8.0 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/graph-gophers/graphql-go v1.3.0 // indirect + github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.0 // indirect + github.com/huin/goupnp v1.0.2 // indirect + github.com/influxdata/influxdb v1.8.3 // indirect + github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/tsdb v0.7.1 // indirect + github.com/rjeczalik/notify v0.9.1 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/crawlers/ldb/go.sum b/crawlers/ldb/go.sum new file mode 100644 index 00000000..a708717d --- /dev/null +++ b/crawlers/ldb/go.sum @@ -0,0 +1,635 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205 h1:UQ7XGjvoOVKGRIuTFXgqGtU/UgMOk8+ikpoHWrWefjQ= +github.com/bugout-dev/humbug/go v0.0.0-20211206230955-57607cd2d205/go.mod h1:U/NXHfc3tzGeQz+xVfpifXdPZi7p6VV8xdP/4ZKeWJU= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= +github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/crawlers/ldb/main.go b/crawlers/ldb/main.go new file mode 100644 index 00000000..44a1eda9 --- /dev/null +++ b/crawlers/ldb/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/bugout-dev/moonstream/crawlers/ldb/cmd" +) + +func main() { + cmd.LDBCLI() +} diff --git a/crawlers/ldb/sample.env b/crawlers/ldb/sample.env new file mode 100644 index 00000000..449b363a --- /dev/null +++ b/crawlers/ldb/sample.env @@ -0,0 +1,3 @@ +export MOONSTREAM_DB_URI="postgresql://:@:/" +export HUMBUG_LDB_CLIENT_ID="" +export HUMBUG_LDB_TOKEN="" diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index a6c50cf2..9724ff01 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -1,6 +1,7 @@ """ The Mooncrawl HTTP API """ +from cgi import test from datetime import datetime, timedelta import logging from os import times @@ -11,6 +12,7 @@ from uuid import UUID import boto3 # type: ignore from fastapi import FastAPI, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy import text from bugout.data import BugoutResource, BugoutResources @@ -21,10 +23,12 @@ from .settings import ( ORIGINS, bugout_client as bc, BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, + MOONSTREAM_S3_QUERIES_BUCKET, + MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from .version import MOONCRAWL_VERSION -from .stats_worker import dashboard +from .stats_worker import dashboard, queries logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -118,7 +122,9 @@ async def status_handler( ) except Exception as e: - logger.error(f"Unhandled status exception, error: {e}") + logger.error( + f"Unhandled /jobs/stats_update start background task exception, error: {e}" + ) raise MoonstreamHTTPException(status_code=500) presigned_urls_response: Dict[UUID, Any] = {} @@ -166,3 +172,68 @@ async def status_handler( ) return presigned_urls_response + + +@app.post("/jobs/{query_id}/query_update", tags=["jobs"]) +async def queries_data_update_handler( + query_id: str, + request_data: data.QueryDataUpdate, + background_tasks: BackgroundTasks, +) -> Dict[str, Any]: + + s3_client = boto3.client("s3") + + expected_query_parameters = text(request_data.query)._bindparams.keys() + + # request.params validations + passed_params = { + key: value + for key, value in request_data.params.items() + if key in expected_query_parameters + } + + if len(passed_params) != len(expected_query_parameters): + logger.error( + f"Unmatched amount of applying query parameters: {passed_params}, query_id:{query_id}." + ) + raise MoonstreamHTTPException( + status_code=500, detail="Unmatched amount of applying query parameters" + ) + + try: + valid_query = queries.query_validation(request_data.query) + except queries.QueryNotValid: + logger.error(f"Incorrect query provided with id: {query_id}") + raise MoonstreamHTTPException( + status_code=401, detail="Incorrect query provided" + ) + except Exception as e: + logger.error(f"Unhandled query execute exception, error: {e}") + raise MoonstreamHTTPException(status_code=500) + + try: + + background_tasks.add_task( + queries.data_generate, + bucket=MOONSTREAM_S3_QUERIES_BUCKET, + query_id=f"{query_id}", + file_type=request_data.file_type, + query=valid_query, + params=request_data.params, + ) + + except Exception as e: + logger.error(f"Unhandled query execute exception, error: {e}") + raise MoonstreamHTTPException(status_code=500) + + stats_presigned_url = s3_client.generate_presigned_url( + "get_object", + Params={ + "Bucket": MOONSTREAM_S3_QUERIES_BUCKET, + "Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}", + }, + ExpiresIn=43200, # 12 hours + HttpMethod="GET", + ) + + return {"url": stats_presigned_url} diff --git a/crawlers/mooncrawl/mooncrawl/data.py b/crawlers/mooncrawl/mooncrawl/data.py index 4bf694c9..ec0b1859 100644 --- a/crawlers/mooncrawl/mooncrawl/data.py +++ b/crawlers/mooncrawl/mooncrawl/data.py @@ -1,9 +1,9 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import List +from typing import List, Any, Dict -from pydantic import BaseModel +from pydantic import BaseModel, Field class AvailableBlockchainType(Enum): @@ -47,3 +47,10 @@ class NowResponse(BaseModel): """ epoch_time: float + + +class QueryDataUpdate(BaseModel): + + file_type: str + query: str + params: Dict[str, Any] = Field(default_factory=dict) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 89d2751f..13494c55 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -84,3 +84,17 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "": raise ValueError( "MOONSTREAM_MOONWORM_TASKS_JOURNAL environment variable must be set" ) + + +MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "") +if MOONSTREAM_S3_QUERIES_BUCKET == "": + raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set") + + +MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", "" +) +if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "": + raise ValueError( + "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set" + ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py new file mode 100644 index 00000000..86a8b419 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -0,0 +1,95 @@ +import csv +import json +import logging +import re +from io import StringIO +from typing import Any, Dict, Optional + +import boto3 # type: ignore +from moonstreamdb.db import yield_db_read_only_session_ctx + +from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +QUERY_REGEX = re.compile("[\[\]@#$%^&?;`/]") + + +class QueryNotValid(Exception): + """ + Raised when query validation not passed. + """ + + +def push_statistics(s3: Any, data: Any, key: str, bucket: str) -> None: + + s3.put_object( + Body=data, + Bucket=bucket, + Key=key, + ContentType="application/json", + Metadata={"drone_query": "data"}, + ) + + logger.info(f"Statistics push to bucket: s3://{bucket}/{key}") + + +def query_validation(query: str) -> str: + """ + Sanitize provided query. + """ + if QUERY_REGEX.search(query) != None: + raise QueryNotValid("Query contains restricted symbols") + + return query + + +def data_generate( + bucket: str, + query_id: str, + file_type: str, + query: str, + params: Optional[Dict[str, Any]], +): + """ + Generate query and push it to S3 + """ + s3 = boto3.client("s3") + + with yield_db_read_only_session_ctx() as db_session: + + if file_type == "csv": + csv_buffer = StringIO() + csv_writer = csv.writer(csv_buffer, delimiter=";") + + # engine.execution_options(stream_results=True) + result = db_session.execute(query, params).keys() + + csv_writer.writerow(result.keys()) + csv_writer.writerows(result.fetchAll()) + + push_statistics( + s3=s3, + data=csv_buffer.getvalue().encode("utf-8"), + key=f"queries/{query_id}/data.{file_type}", + bucket=bucket, + ) + else: + block_number, block_timestamp = db_session.execute( + "SELECT block_number, block_timestamp FROM polygon_labels WHERE block_number=(SELECT max(block_number) FROM polygon_labels where label='moonworm-alpha') limit 1;", + ).one() + + data = json.dumps( + { + "block_number": block_number, + "block_timestamp": block_timestamp, + "data": [dict(row) for row in db_session.execute(query, params)], + } + ).encode("utf-8") + push_statistics( + s3=s3, + data=data, + key=f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}", + bucket=bucket, + ) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py new file mode 100644 index 00000000..a71911d7 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/test_queries.py @@ -0,0 +1,65 @@ +import unittest + +from . import queries + + +class TestQueries(unittest.TestCase): + def test_query_validation(self): + q = "SELECT * FROM ethereum_blocks" + self.assertEqual(queries.query_validation(q), q) + + q = "select count(*), tx_dublicates from ( select count(*) as tx_dublicates from polygon_labels where address = '0x123' and label_data->>'name' = 'Transfer' group by transaction_hash, log_index order by tx_dublicates desc) as dublicates group by dublicates.tx_dublicates" + self.assertEqual(queries.query_validation(q), q) + + q = """ + Select difference.address, + difference.transfers_in - difference.transfers_out as count + from ( + SELECT total.address, + sum(total.transfer_out) as transfers_out, + sum(total.transfer_in) as transfers_in + from ( + SELECT label_data->'args'->>'from' as address, + (label_data->'args'->'values'->>0)::int as transfer_out, + 0 as transfer_in + from polygon_labels + where address = '0x123' + and label = 'moonworm' + and label_data->>'name' = 'TransferBatch' + and (label_data->'args'->'ids'->>0)::int in (2, 3, 4) + UNION ALL + SELECT label_data->'args'->>'from' as address, + (label_data->'args'->'value')::int as transfer_out, + 0 as transfer_in + from polygon_labels + where address = '0x123' + and label = 'moonworm' + and label_data->>'name' = 'TransferSingle' + and (label_data->'args'->>'id')::int in (2, 3, 4) + ) as total + group by address + ) difference + order by count desc + """ + self.assertEqual(queries.query_validation(q), q) + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("SELECT hash FROM ethereum_transaction;") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("%20UNION") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("?id=1") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("FROM`") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("WHERE login='[USER]'") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("OR(1=1)#") + + with self.assertRaises(queries.QueryNotValid): + queries.query_validation("/etc/hosts") diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 48874ac7..586fbde4 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.1.3" +MOONCRAWL_VERSION = "0.1.4" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a22bb6b7..57871f7e 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -5,13 +5,16 @@ export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://=0.1.19", "chardet", "fastapi", - "moonstreamdb>=0.2.2", + "moonstreamdb>=0.2.3", "moonworm==0.1.11", "humbug", "pydantic", diff --git a/db/configs/docker_generate_env.bash b/db/configs/docker_generate_env.bash index 0ef9c16f..306a1d3b 100755 --- a/db/configs/docker_generate_env.bash +++ b/db/configs/docker_generate_env.bash @@ -41,6 +41,7 @@ sed --in-place 's|^export * ||' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed --in-place 's|"||g' "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" sed -i "s|^MOONSTREAM_DB_URI=.*|MOONSTREAM_DB_URI=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" +sed -i "s|^MOONSTREAM_DB_URI_READ_ONLY=.*|MOONSTREAM_DB_URI_READ_ONLY=$DOCKER_MOONSTREAMDB_DB_URI|" "$SCRIPT_DIR/$DOCKER_MOONSTREAMDB_ENV_FILE" # Generate alembic config diff --git a/db/configs/sample.env b/db/configs/sample.env index c5409401..8cfb9a45 100644 --- a/db/configs/sample.env +++ b/db/configs/sample.env @@ -1,3 +1,4 @@ # Required environment variables to work with database CLI export MOONSTREAM_DB_URI="postgresql://:@:/" +export MOONSTREAM_DB_URI_READ_ONLY="postgresql://:@:/" export MOONSTREAM_POOL_SIZE=0 diff --git a/db/moonstreamdb/db.py b/db/moonstreamdb/db.py index bfe055e9..4e6c7383 100644 --- a/db/moonstreamdb/db.py +++ b/db/moonstreamdb/db.py @@ -11,6 +11,10 @@ MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") if MOONSTREAM_DB_URI is None: raise ValueError("MOONSTREAM_DB_URI environment variable must be set") +MOONSTREAM_DB_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_URI_READ_ONLY") +if MOONSTREAM_DB_URI_READ_ONLY is None: + raise ValueError("MOONSTREAM_DB_URI_READ_ONLY environment variable must be set") + MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE", 0) try: if MOONSTREAM_POOL_SIZE_RAW is not None: @@ -60,3 +64,30 @@ def yield_db_session() -> Session: yield_db_session_ctx = contextmanager(yield_db_session) + +# Read only + +RO_engine = create_engine( + MOONSTREAM_DB_URI_READ_ONLY, + pool_size=MOONSTREAM_POOL_SIZE, + connect_args={ + "options": f"-c statement_timeout={MOONSTREAM_DB_STATEMENT_TIMEOUT_MILLIS}" + }, +) +RO_SessionLocal = sessionmaker(bind=RO_engine) + + +def yield_db_read_only_session() -> Session: + """ + Yields a database connection (created using environment variables). + As per FastAPI docs: + https://fastapi.tiangolo.com/tutorial/sql-databases/#create-a-dependency + """ + session = RO_SessionLocal() + try: + yield session + finally: + session.close() + + +yield_db_read_only_session_ctx = contextmanager(yield_db_read_only_session) diff --git a/db/moonstreamdb/version.py b/db/moonstreamdb/version.py index 7f3d3eaa..80ecc71f 100644 --- a/db/moonstreamdb/version.py +++ b/db/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.2.2" +MOONSTREAMDB_VERSION = "0.2.3" diff --git a/frontend/pages/index.js b/frontend/pages/index.js index b5d2c269..ba3c0f76 100644 --- a/frontend/pages/index.js +++ b/frontend/pages/index.js @@ -128,6 +128,7 @@ const assets = { cryptoTraders: `${AWS_ASSETS_PATH}/crypto+traders.png`, smartDevelopers: `${AWS_ASSETS_PATH}/smart+contract+developers.png`, cointelegraph: `${AWS_ASSETS_PATH}/featured_by/Cointelegraph_logo.png`, + forte: `${AWS_ASSETS_PATH}/featured_by/forte_logo.png`, educativesessions: `${AWS_ASSETS_PATH}/featured_by/educative_logo.png`, cryptoinsiders: `${AWS_ASSETS_PATH}/featured_by/crypto_insiders.png`, cryptoslate: `${AWS_ASSETS_PATH}/featured_by/cs-media-logo-light.png`, @@ -135,7 +136,7 @@ const assets = { ethereumBlackLogo: `${AWS_ASSETS_PATH}/eth-diamond-black.png`, ethereumRainbowLogo: `${AWS_ASSETS_PATH}/eth-diamond-rainbow.png`, maticLogo: `${AWS_ASSETS_PATH}/matic-token-inverted-icon.png`, - erc20: `${AWS_ASSETS_PATH}/ERC 20.png`, + lender: `${AWS_ASSETS_PATH}/lender.png`, DAO: `${AWS_ASSETS_PATH}/DAO .png`, NFT: `${AWS_ASSETS_PATH}/NFT.png`, bc101: `${AWS_ASSETS_PATH}/featured_by/blockchain101_logo.png`, @@ -503,17 +504,17 @@ const Homepage = () => { - Bridges + Engine - Open up your economy to Ethereum’s rich DeFi ecosystem, - no matter which chain you live on. + Create and manage tokens with custom mechanics specific + to your project. @@ -603,17 +604,26 @@ const Homepage = () => { name="Laguna games" caseURL="" ImgURL={assets["laguna"]} + boxURL="https://laguna.games/" /> + @@ -739,29 +749,28 @@ const Homepage = () => { }} elementName={"element3"} colorScheme="orange" - badge={`Moonstream bridges`} + badge={`Moonstream engine`} bullets={[ { - text: `Make it easy for participants to bring value into your economy from other blockchains.`, + text: `Deploy customizable and upgradable characters, items, and currencies into your economy`, icon: GiCrossedChains, color: "orange.50", bgColor: "orange.900", }, { - text: `Make your tokens and NFTs multi-chain.`, + text: `Monitor interactions between these tokens`, icon: GiChainedHeart, color: "orange.50", bgColor: "orange.900", }, { - text: `Run the cost-intensive parts of your economy off of Ethereum while tapping into Ethereum’s DeFi ecosystem. - `, + text: `Secure the tokens with Moonstream defense bots.`, icon: GiChaingun, color: "orange.50", bgColor: "orange.900", }, ]} - imgURL={assets["erc20"]} + imgURL={assets["lender"]} /> { name="cointelegraph" caseURL="" ImgURL={assets["cointelegraph"]} + boxURL="https://cointelegraph.com/news/17-of-addresses-snapped-up-80-of-all-ethereum-nfts-since-april" /> diff --git a/frontend/pages/team/index.js b/frontend/pages/team/index.js index 508e680d..73ca5348 100644 --- a/frontend/pages/team/index.js +++ b/frontend/pages/team/index.js @@ -248,8 +248,8 @@ const Product = () => { avatarURL={assets["dragonfly"]} name={"Tim Pechersky"} atName={"@peersky"} - content={`Spectral hopper. Hes special ability to precieve world trough spectral domain. Occasionaly - shifts in to time domain to spend some in doing flow + content={`Spectral hopper. Has special ability to perceive world trough spectral domain. Occasionaly + shifts in time domain to spend some in doing flow arts, surfing, and connecting with nature.`} /> { +const TrustedBadge = ({ + name, + caseURL, + ImgURL, + scale, + isGrayScale, + boxURL, +}) => { const _scale = scale ?? 1; return ( - - {name} - {caseURL && ( - // - + + - {`Read more >`} - - // - )} - + {name} + {caseURL && ( + // + + {`Read more >`} + + // + )} + + + ); }; export default TrustedBadge; diff --git a/nodes/deploy/ethereum/deploy.bash b/nodes/deploy/ethereum/deploy.bash index 03d8e17d..75d63649 100755 --- a/nodes/deploy/ethereum/deploy.bash +++ b/nodes/deploy/ethereum/deploy.bash @@ -65,7 +65,6 @@ chmod 644 "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODE_STATUS_SERVER_SERVICE_FILE}" systemctl daemon-reload systemctl restart "${NODE_STATUS_SERVER_SERVICE_FILE}" -systemctl status "${NODE_STATUS_SERVER_SERVICE_FILE}" echo echo diff --git a/nodes/deploy/polygon/deploy.bash b/nodes/deploy/polygon/deploy.bash index 17230994..39acfaa0 100755 --- a/nodes/deploy/polygon/deploy.bash +++ b/nodes/deploy/polygon/deploy.bash @@ -69,7 +69,6 @@ chmod 644 "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${NODE_STATUS_SERVER_SERVICE_FILE}" "/etc/systemd/system/${NODE_STATUS_SERVER_SERVICE_FILE}" systemctl daemon-reload systemctl restart "${NODE_STATUS_SERVER_SERVICE_FILE}" -systemctl status "${NODE_STATUS_SERVER_SERVICE_FILE}" echo echo