mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Cleaned from unused crawler services

pull/939/head
kompotkot 2023-10-19 12:00:52 +00:00
rodzic 6949a4dafe
commit 8051e78f5b
5 zmienionych plików z 0 dodań i 171 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

18
crawlers/deploy/deploy.bash
Wyświetl plik

@ -34,7 +34,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
@ -51,7 +50,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service"
POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
POLYGON_STATE_SERVICE_FILE="polygon-state.service"
POLYGON_STATE_TIMER_FILE="polygon-state.timer"
@ -191,14 +189,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
# echo
# echo
# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}"
# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}"
# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}"
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
@ -270,14 +260,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
# echo
# echo
# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}"
# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}"
# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}"
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"

17
crawlers/deploy/ethereum-txpool.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Ethereum txpool crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain ethereum -access-id "${NB_CONTROLLER_ACCESS_ID}"
CPUWeight=30
SyslogIdentifier=ethereum-txpool
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/moonworm-unicorns-mainnet.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Moonworm CryptoUnicorns watch custom systemd service
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60
CPUWeight=70
SyslogIdentifier=moonworm-unicorns-mainnet
[Install]
WantedBy=multi-user.target

102
crawlers/deploy/parameters.py
Wyświetl plik

@ -1,102 +0,0 @@
"""
Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
"""
import argparse
from dataclasses import dataclass
import sys
from typing import Any, Dict, Iterable, List, Optional
import boto3
@dataclass
class EnvironmentVariable:
name: str
value: str
def get_parameters(path: str) -> List[Dict[str, Any]]:
"""
Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters.
Relies on the appropriate environment variables to authenticate against AWS:
https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
"""
ssm = boto3.client("ssm")
next_token: Optional[bool] = True
parameters: List[Dict[str, Any]] = []
while next_token is not None:
kwargs = {"Path": path, "Recursive": False, "WithDecryption": True}
if next_token is not True:
kwargs["NextToken"] = next_token
response = ssm.get_parameters_by_path(**kwargs)
new_parameters = response.get("Parameters", [])
parameters.extend(new_parameters)
next_token = response.get("NextToken")
return parameters
def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable:
"""
Transforms parameters returned by the AWS SSM API into EnvironmentVariables.
"""
parameter_path = parameter_object.get("Name")
if parameter_path is None:
raise ValueError('Did not find "Name" in parameter object')
name = parameter_path.split("/")[-1].upper()
value = parameter_object.get("Value")
if value is None:
raise ValueError('Did not find "Value" in parameter object')
return EnvironmentVariable(name, value)
def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str:
"""
Produces a string which, when executed in a shell, exports the desired environment variables as
specified by env_vars.
"""
prefix = "export " if with_export else ""
return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars])
def extract_handler(args: argparse.Namespace) -> None:
"""
Save environment variables to file.
"""
result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export)
with args.outfile as ofp:
print(result, file=ofp)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Materialize environment variables from AWS SSM Parameter Store"
)
parser.set_defaults(func=lambda _: parser.print_help())
subcommands = parser.add_subparsers(description="Parameters commands")
parser_extract = subcommands.add_parser(
"extract", description="Parameters extract commands"
)
parser_extract.set_defaults(func=lambda _: parser_extract.print_help())
parser_extract.add_argument(
"-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout
)
parser_extract.add_argument(
"--export",
action="store_true",
help="Set to output environment strings with export statements",
)
parser_extract.add_argument(
"-p",
"--path",
default=None,
help="SSM path from which to pull environment variables (pull is NOT recursive)",
)
parser_extract.set_defaults(func=extract_handler)
args = parser.parse_args()
args.func(args)

17
crawlers/deploy/polygon-txpool.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=Polygon txpool crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain polygon -access-id "${NB_CONTROLLER_ACCESS_ID}"
CPUWeight=30
SyslogIdentifier=polygon-txpool
[Install]
WantedBy=multi-user.target