From 8051e78f5b3b5c4c6126bf54e301b9d1ff08de66 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 19 Oct 2023 12:00:52 +0000 Subject: [PATCH 1/2] Cleaned from unused crawler services --- crawlers/deploy/deploy.bash | 18 ---- crawlers/deploy/ethereum-txpool.service | 17 --- .../deploy/moonworm-unicorns-mainnet.service | 17 --- crawlers/deploy/parameters.py | 102 ------------------ crawlers/deploy/polygon-txpool.service | 17 --- 5 files changed, 171 deletions(-) delete mode 100644 crawlers/deploy/ethereum-txpool.service delete mode 100644 crawlers/deploy/moonworm-unicorns-mainnet.service delete mode 100644 crawlers/deploy/parameters.py delete mode 100644 crawlers/deploy/polygon-txpool.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index fc32785a..80a6f265 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -34,7 +34,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer" -ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service" ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer" ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service" @@ -51,7 +50,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service" POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer" -POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service" POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service" POLYGON_STATE_SERVICE_FILE="polygon-state.service" POLYGON_STATE_TIMER_FILE="polygon-state.timer" @@ -191,14 +189,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}" @@ -270,14 +260,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/ethereum-txpool.service b/crawlers/deploy/ethereum-txpool.service deleted file mode 100644 index 39e6d280..00000000 --- a/crawlers/deploy/ethereum-txpool.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Ethereum txpool crawler -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain ethereum -access-id "${NB_CONTROLLER_ACCESS_ID}" -CPUWeight=30 -SyslogIdentifier=ethereum-txpool - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/moonworm-unicorns-mainnet.service b/crawlers/deploy/moonworm-unicorns-mainnet.service deleted file mode 100644 index 67448c67..00000000 --- a/crawlers/deploy/moonworm-unicorns-mainnet.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Moonworm CryptoUnicorns watch custom systemd service -StartLimitIntervalSec=300 -StartLimitBurst=3 -After=network.target - -[Service] -Restart=on-failure -RestartSec=15s -WorkingDirectory=/home/ubuntu -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonworm-env/bin/python -m moonworm.cli watch-cu -w "${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}?access_id=${NB_CONTROLLER_ACCESS_ID}&data_source=blockchain" -c 0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f -d 21418707 --confirmations 60 -CPUWeight=70 -SyslogIdentifier=moonworm-unicorns-mainnet - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/parameters.py b/crawlers/deploy/parameters.py deleted file mode 100644 index c8df5797..00000000 --- a/crawlers/deploy/parameters.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Collect secrets from AWS SSM Parameter Store and output as environment variable exports. -""" -import argparse -from dataclasses import dataclass -import sys -from typing import Any, Dict, Iterable, List, Optional - -import boto3 - - -@dataclass -class EnvironmentVariable: - name: str - value: str - - -def get_parameters(path: str) -> List[Dict[str, Any]]: - """ - Retrieve parameters from AWS SSM Parameter Store. Decrypts any encrypted parameters. - - Relies on the appropriate environment variables to authenticate against AWS: - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - """ - ssm = boto3.client("ssm") - next_token: Optional[bool] = True - parameters: List[Dict[str, Any]] = [] - while next_token is not None: - kwargs = {"Path": path, "Recursive": False, "WithDecryption": True} - if next_token is not True: - kwargs["NextToken"] = next_token - response = ssm.get_parameters_by_path(**kwargs) - new_parameters = response.get("Parameters", []) - parameters.extend(new_parameters) - next_token = response.get("NextToken") - - return parameters - - -def parameter_to_env(parameter_object: Dict[str, Any]) -> EnvironmentVariable: - """ - Transforms parameters returned by the AWS SSM API into EnvironmentVariables. - """ - parameter_path = parameter_object.get("Name") - if parameter_path is None: - raise ValueError('Did not find "Name" in parameter object') - name = parameter_path.split("/")[-1].upper() - - value = parameter_object.get("Value") - if value is None: - raise ValueError('Did not find "Value" in parameter object') - - return EnvironmentVariable(name, value) - - -def env_string(env_vars: Iterable[EnvironmentVariable], with_export: bool) -> str: - """ - Produces a string which, when executed in a shell, exports the desired environment variables as - specified by env_vars. - """ - prefix = "export " if with_export else "" - return "\n".join([f'{prefix}{var.name}="{var.value}"' for var in env_vars]) - - -def extract_handler(args: argparse.Namespace) -> None: - """ - Save environment variables to file. - """ - result = env_string(map(parameter_to_env, get_parameters(args.path)), args.export) - with args.outfile as ofp: - print(result, file=ofp) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Materialize environment variables from AWS SSM Parameter Store" - ) - parser.set_defaults(func=lambda _: parser.print_help()) - subcommands = parser.add_subparsers(description="Parameters commands") - - parser_extract = subcommands.add_parser( - "extract", description="Parameters extract commands" - ) - parser_extract.set_defaults(func=lambda _: parser_extract.print_help()) - parser_extract.add_argument( - "-o", "--outfile", type=argparse.FileType("w"), default=sys.stdout - ) - parser_extract.add_argument( - "--export", - action="store_true", - help="Set to output environment strings with export statements", - ) - parser_extract.add_argument( - "-p", - "--path", - default=None, - help="SSM path from which to pull environment variables (pull is NOT recursive)", - ) - parser_extract.set_defaults(func=extract_handler) - - args = parser.parse_args() - args.func(args) diff --git a/crawlers/deploy/polygon-txpool.service b/crawlers/deploy/polygon-txpool.service deleted file mode 100644 index f4f871bc..00000000 --- a/crawlers/deploy/polygon-txpool.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Polygon txpool crawler -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/txpool -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream/crawlers/txpool/txpool -blockchain polygon -access-id "${NB_CONTROLLER_ACCESS_ID}" -CPUWeight=30 -SyslogIdentifier=polygon-txpool - -[Install] -WantedBy=multi-user.target \ No newline at end of file From fa3caefe06d0ee50f5ab708456d3e53f468f0949 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 19 Oct 2023 13:20:09 +0000 Subject: [PATCH 2/2] Fixed naming of systemd service to correlate with files and deploy --- crawlers/deploy/ethereum-historical-crawl-events.service | 2 +- crawlers/deploy/ethereum-historical-crawl-transactions.service | 2 +- crawlers/deploy/mumbai-historical-crawl-events.service | 2 +- crawlers/deploy/mumbai-historical-crawl-transactions.service | 2 +- crawlers/deploy/polygon-historical-crawl-events.service | 2 +- crawlers/deploy/polygon-historical-crawl-transactions.service | 2 +- crawlers/deploy/wyrm-historical-crawl-events.service | 2 +- crawlers/deploy/wyrm-historical-crawl-transactions.service | 2 +- crawlers/deploy/xdai-historical-crawl-events.service | 2 +- crawlers/deploy/xdai-historical-crawl-transactions.service | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crawlers/deploy/ethereum-historical-crawl-events.service b/crawlers/deploy/ethereum-historical-crawl-events.service index 006a03c4..881d21b2 100644 --- a/crawlers/deploy/ethereum-historical-crawl-events.service +++ b/crawlers/deploy/ethereum-historical-crawl-events.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 -SyslogIdentifier=ethereum-historical-crawler-events +SyslogIdentifier=ethereum-historical-crawl-events [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/ethereum-historical-crawl-transactions.service b/crawlers/deploy/ethereum-historical-crawl-transactions.service index fa9fcd28..25e11793 100644 --- a/crawlers/deploy/ethereum-historical-crawl-transactions.service +++ b/crawlers/deploy/ethereum-historical-crawl-transactions.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 -SyslogIdentifier=ethereum-historical-crawler-transactions +SyslogIdentifier=ethereum-historical-crawl-transactions [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/mumbai-historical-crawl-events.service b/crawlers/deploy/mumbai-historical-crawl-events.service index 488a1a94..cc56464e 100644 --- a/crawlers/deploy/mumbai-historical-crawl-events.service +++ b/crawlers/deploy/mumbai-historical-crawl-events.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 -SyslogIdentifier=mumbai-historical-crawler-events +SyslogIdentifier=mumbai-historical-crawl-events [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/mumbai-historical-crawl-transactions.service b/crawlers/deploy/mumbai-historical-crawl-transactions.service index 968d88bb..9a88ac2f 100644 --- a/crawlers/deploy/mumbai-historical-crawl-transactions.service +++ b/crawlers/deploy/mumbai-historical-crawl-transactions.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 -SyslogIdentifier=mumbai-historical-crawler-transactions +SyslogIdentifier=mumbai-historical-crawl-transactions [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/polygon-historical-crawl-events.service b/crawlers/deploy/polygon-historical-crawl-events.service index c8017a93..965e6747 100644 --- a/crawlers/deploy/polygon-historical-crawl-events.service +++ b/crawlers/deploy/polygon-historical-crawl-events.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 -SyslogIdentifier=polygon-historical-crawler-events +SyslogIdentifier=polygon-historical-crawl-events [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/polygon-historical-crawl-transactions.service b/crawlers/deploy/polygon-historical-crawl-transactions.service index d8229b02..36639e3e 100644 --- a/crawlers/deploy/polygon-historical-crawl-transactions.service +++ b/crawlers/deploy/polygon-historical-crawl-transactions.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 -SyslogIdentifier=polygon-historical-crawler-transactions +SyslogIdentifier=polygon-historical-crawl-transactions [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/wyrm-historical-crawl-events.service b/crawlers/deploy/wyrm-historical-crawl-events.service index cd73d8c0..7ecefc11 100644 --- a/crawlers/deploy/wyrm-historical-crawl-events.service +++ b/crawlers/deploy/wyrm-historical-crawl-events.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 -SyslogIdentifier=wyrm-historical-crawler-events +SyslogIdentifier=wyrm-historical-crawl-events [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/wyrm-historical-crawl-transactions.service b/crawlers/deploy/wyrm-historical-crawl-transactions.service index 0ef17a88..41186544 100644 --- a/crawlers/deploy/wyrm-historical-crawl-transactions.service +++ b/crawlers/deploy/wyrm-historical-crawl-transactions.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 -SyslogIdentifier=wyrm-historical-crawler-transactions +SyslogIdentifier=wyrm-historical-crawl-transactions [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/xdai-historical-crawl-events.service b/crawlers/deploy/xdai-historical-crawl-events.service index 11b0fa5a..66d56405 100644 --- a/crawlers/deploy/xdai-historical-crawl-events.service +++ b/crawlers/deploy/xdai-historical-crawl-events.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 -SyslogIdentifier=xdai-historical-crawler-events +SyslogIdentifier=xdai-historical-crawl-events [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/xdai-historical-crawl-transactions.service b/crawlers/deploy/xdai-historical-crawl-transactions.service index 48b37eeb..20a3c616 100644 --- a/crawlers/deploy/xdai-historical-crawl-transactions.service +++ b/crawlers/deploy/xdai-historical-crawl-transactions.service @@ -11,7 +11,7 @@ Restart=on-failure RestartSec=15s ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 -SyslogIdentifier=xdai-historical-crawler-transactions +SyslogIdentifier=xdai-historical-crawl-transactions [Install] WantedBy=multi-user.target \ No newline at end of file