mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Added missed env variables in sample.env

pull/550/head
kompotkot 2022-03-09 12:44:54 +00:00
rodzic 57faaf814e
commit b408c20b0b
7 zmienionych plików z 41 dodań i 37 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

7
backend/configs/sample.env
Wyświetl plik

@ -8,18 +8,21 @@ export MOONSTREAM_APPLICATION_ID="<issued_bugout_application_id>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Access_token_to_application_resources>"
export MOONSTREAM_POOL_SIZE=0
export MOONSTREAM_MOONWORM_TASKS_JOURNAL="<Bugout_journal_with_tasks_for_moonworm>"
export MOONSTREAM_CRAWLERS_SERVER_URL="<Moonstream_crawlers_server_API_endpoint_URL>"
export MOONSTREAM_CRAWLERS_SERVER_PORT="<Moonstream_crawlers_server_port>"
# Blockchain, txpool, whalewatch data depends variables
export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
export HUMBUG_TXPOOL_CLIENT_ID="<Bugout_Humbug_client_id_for_txpool_transactions_in_journal>"
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
export MOONSTREAM_QUERIES_JOURNAL_ID="<bugout_journal_id_where_store_queries_for_executingw>"
export MOONSTREAM_QUERIES_JOURNAL_ID="<bugout_journal_id_where_store_queries_for_executing>"
# Set following parameters if AWS node instance and S3 smartcontracts configured
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_to_store_smart_contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS_S3_bucket_to_store_smart_contracts_ABI>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx_for_AWS_S3_bucket_(prod,dev,..)>"
export MOONSTREAM_S3_QUERIES_BUCKET="<AWS_S3_bucket_to_store_sql_queries>"
export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev"
# Set the following variables in the most reasonable manner for your development environment
export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout_umbug_token_for_crash_reports>"

8
backend/moonstreamapi/routes/queries.py
Wyświetl plik

@ -20,8 +20,8 @@ from ..settings import (
MOONSTREAM_APPLICATION_ID,
MOONSTREAM_CRAWLERS_SERVER_URL,
MOONSTREAM_CRAWLERS_SERVER_PORT,
MOONSTREAM_QUERIES_BUCKET,
MOONSTREAM_QUERIES_BUCKET_PREFIX,
MOONSTREAM_S3_QUERIES_BUCKET,
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
MOONSTREAM_QUERIES_JOURNAL_ID,
)
from ..settings import bugout_client as bc
@ -345,8 +345,8 @@ async def get_access_link_handler(
stats_presigned_url = s3.generate_presigned_url(
"get_object",
Params={
"Bucket": MOONSTREAM_QUERIES_BUCKET,
"Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}",
"Bucket": MOONSTREAM_S3_QUERIES_BUCKET,
"Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}",
},
ExpiresIn=300000,
HttpMethod="GET",

14
backend/moonstreamapi/settings.py
Wyświetl plik

@ -100,14 +100,14 @@ if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "":
"MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable must be set"
)
MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "")
if MOONSTREAM_QUERIES_BUCKET == "":
raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set")
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
if MOONSTREAM_S3_QUERIES_BUCKET == "":
raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set")
MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get(
"MOONSTREAM_QUERIES_BUCKET_PREFIX", ""
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get(
"MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", ""
)
if MOONSTREAM_QUERIES_BUCKET_PREFIX == "":
if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "":
raise ValueError(
"MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set"
"MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set"
)

10
crawlers/mooncrawl/mooncrawl/api.py
Wyświetl plik

@ -23,8 +23,8 @@ from .settings import (
ORIGINS,
bugout_client as bc,
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,
MOONSTREAM_QUERIES_BUCKET,
MOONSTREAM_QUERIES_BUCKET_PREFIX,
MOONSTREAM_S3_QUERIES_BUCKET,
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
)
from .version import MOONCRAWL_VERSION
@ -204,7 +204,7 @@ async def queries_data_update_handler(
background_tasks.add_task(
queries.data_generate,
bucket=MOONSTREAM_QUERIES_BUCKET,
bucket=MOONSTREAM_S3_QUERIES_BUCKET,
query_id=f"{query_id}",
file_type=request_data.file_type,
query=request_data.query,
@ -218,8 +218,8 @@ async def queries_data_update_handler(
stats_presigned_url = s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket": MOONSTREAM_QUERIES_BUCKET,
"Key": f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}",
"Bucket": MOONSTREAM_S3_QUERIES_BUCKET,
"Key": f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{request_data.file_type}",
},
ExpiresIn=43200, # 12 hours
HttpMethod="GET",

14
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -86,15 +86,15 @@ if MOONSTREAM_MOONWORM_TASKS_JOURNAL == "":
)
MOONSTREAM_QUERIES_BUCKET = os.environ.get("MOONSTREAM_QUERIES_BUCKET", "")
if MOONSTREAM_QUERIES_BUCKET == "":
raise ValueError("MOONSTREAM_QUERIES_BUCKET environment variable must be set")
MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
if MOONSTREAM_S3_QUERIES_BUCKET == "":
raise ValueError("MOONSTREAM_S3_QUERIES_BUCKET environment variable must be set")
MOONSTREAM_QUERIES_BUCKET_PREFIX = os.environ.get(
"MOONSTREAM_QUERIES_BUCKET_PREFIX", ""
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX = os.environ.get(
"MOONSTREAM_S3_QUERIES_BUCKET_PREFIX", ""
)
if MOONSTREAM_QUERIES_BUCKET_PREFIX == "":
if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "":
raise ValueError(
"MOONSTREAM_QUERIES_BUCKET_PREFIX environment variable must be set"
"MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set"
)

4
crawlers/mooncrawl/mooncrawl/stats_worker/queries.py
Wyświetl plik

@ -7,7 +7,7 @@ from typing import Any, Dict, Optional
import boto3 # type: ignore
from moonstreamdb.db import yield_db_read_only_session_ctx
from ..settings import MOONSTREAM_QUERIES_BUCKET_PREFIX
from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -71,6 +71,6 @@ def data_generate(
push_statistics(
s3=s3,
data=data,
key=f"{MOONSTREAM_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}",
key=f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/data.{file_type}",
bucket=bucket,
)

21
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -6,14 +6,15 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_pol
export MOONSTREAM_CRAWL_WORKERS=4
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_DB_URI_READ_ONLY="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (prod,dev,..)>"
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"
export MOONSTREAM_DATA_JOURNAL_ID="<Bugout journal id for moonstream>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout access token for moonstream>"
export NFT_HUMBUG_TOKEN="<Token for nft crawler>"
export MOONSTREAM_ETHERSCAN_TOKEN="<Token_for_etherscan>"
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"
export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx_for_AWS_S3_bucket_(prod,dev,..)>"
export MOONSTREAM_HUMBUG_TOKEN="<Token_for_crawlers_store_data_via_Humbug>"
export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
export MOONSTREAM_DATA_JOURNAL_ID="<Bugout_journal_id_for_moonstream>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout_access_token_for_moonstream>"
export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
export MOONSTREAM_MOONWORM_TASKS_JOURNAL="<journal_with_tasks_for_moonworm_crawler>"
export MOONSTREAM_QUERIES_BUCKET="<bucket for queries data>"
export MOONSTREAM_S3_QUERIES_BUCKET="<AWS_S3_bucket_to_store_sql_queries>"
export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev"