From 0bb88788ca8330572da90d3b5f926b6a7b4ecb8f Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 2 Sep 2023 19:01:02 +0300 Subject: [PATCH 1/9] Add changes. --- moonstreamapi/moonstreamapi/actions.py | 6 +++--- moonstreamapi/moonstreamapi/admin/cli.py | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 841af7b5..e75e5654 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -470,7 +470,7 @@ def upload_abi_to_s3( def get_all_entries_from_search( - journal_id: str, search_query: str, limit: int, token: str + journal_id: str, search_query: str, limit: int, token: str, content: bool = False ) -> List[BugoutSearchResult]: """ Get all required entries from journal using search interface @@ -483,7 +483,7 @@ def get_all_entries_from_search( token=token, journal_id=journal_id, query=search_query, - content=False, + content=content, timeout=10.0, limit=limit, offset=offset, @@ -496,7 +496,7 @@ def get_all_entries_from_search( token=token, journal_id=journal_id, query=search_query, - content=False, + content=content, timeout=10.0, limit=limit, offset=offset, diff --git a/moonstreamapi/moonstreamapi/admin/cli.py b/moonstreamapi/moonstreamapi/admin/cli.py index b05a595a..84b14b1e 100644 --- a/moonstreamapi/moonstreamapi/admin/cli.py +++ b/moonstreamapi/moonstreamapi/admin/cli.py @@ -20,6 +20,7 @@ from .migrations import ( checksum_address, update_dashboard_subscription_key, generate_entity_subscriptions, + add_selectors, ) @@ -87,6 +88,9 @@ steps: - id: 20230501 name: fix_duplicates_keys_in_entity_subscription description: Fix entity duplicates keys for all subscriptions introduced in 20230213 +- id: 20230904 +name fill_missing_selectors_in_moonworm_tasks +description: Get all moonworm jobs from moonworm journal and add selector tag if it not represent """ logger.info(entity_migration_overview) @@ -117,6 +121,23 @@ def migrations_run(args: argparse.Namespace) -> None: web3_session = yield_web3_provider() db_session = SessionLocal() try: + if args.id == 20230904: + step_order = ["fill_missing_selectors_in_moonworm_tasks"] + step_map: Dict[str, Dict[str, Any]] = { + "upgrade": { + "fill_missing_selectors_in_moonworm_tasks": { + "action": add_selectors.fill_missing_selectors_in_moonworm_tasks, + "description": "Get all moonworm jobs from moonworm journal and add selector tag if it not represent", + }, + }, + "downgrade": {}, + } + if args.command not in ["upgrade", "downgrade"]: + logger.info("Wrong command. Please use upgrade or downgrade") + step = args.step + + migration_run(step_map, args.command, step, step_order) + if args.id == 20230501: # fix entity duplicates keys for all subscriptions introduced in 20230213 @@ -227,7 +248,7 @@ def moonworm_tasks_add_subscription_handler(args: argparse.Namespace) -> None: def main() -> None: cli_description = f"""Moonstream Admin CLI -Please make sure that the following environment variables are set in your environment and exported to +Please m35ake sure that the following environment variables are set in your environment and exported to subprocesses: 1. MOONSTREAM_APPLICATION_ID 2. MOONSTREAM_ADMIN_ACCESS_TOKEN From 8502889540693aad69895196707256eddb17ca58 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 2 Sep 2023 19:01:36 +0300 Subject: [PATCH 2/9] Add migration. --- .../admin/migrations/add_selectors.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py new file mode 100644 index 00000000..de0b96ea --- /dev/null +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -0,0 +1,84 @@ +""" +Add selectors to all moonworm tasks. +""" +import logging +import json + + +from bugout.exceptions import BugoutResponseException +from web3 import Web3 + +from ...settings import ( + BUGOUT_REQUEST_TIMEOUT_SECONDS, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_MOONWORM_TASKS_JOURNAL, +) +from ...settings import bugout_client as bc +from ...actions import get_all_entries_from_search + +logger = logging.getLogger(__name__) + + +def fill_missing_selectors_in_moonworm_tasks() -> None: + """ + Add selectors to all moonworm tasks. + """ + + batch_size = 100 + + moonworm_tasks = get_all_entries_from_search( + journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + search_query="type:subscription !#version:2.0", + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + + entries_tags = [] + + ## batch tasks + + for task_batch in [ + moonworm_tasks[i : i + batch_size] + for i in range(0, len(moonworm_tasks), batch_size) + ]: + for task in task_batch: + tags = ["#version:2.0"] + + ## get abi + try: + abi = json.loads(task.content) + except Exception as e: + logger.warn(f"Unable to parse abi from task: {task.id}") + continue + + if not any([tag.startswith("selector:") for tag in task.tags]): + ## generate selector + + abi_selector = Web3.keccak( + text=abi["name"] + + "(" + + ",".join(map(lambda x: x["type"], abi["inputs"])) + + ")" + )[:4].hex() + + tags.append(f"#selector:{abi_selector}") + + entries_tags.append( + { + "entry_id": task.entry_url.split("/")[-1], ## 😭 + "tags": tags, + } + ) + + ## update entries + + try: + bc.create_entries_tags( + journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + entries_tags=entries_tags, + timeout=15, + ) + except BugoutResponseException as e: + logger.error(f"Unable to update entries tags: {e}") + continue From 2719d9baba56bf62277750f00e60694ac45cc8ca Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 7 Sep 2023 18:04:34 +0300 Subject: [PATCH 3/9] Add changes. --- moonstreamapi/moonstreamapi/actions.py | 25 ++++++++++++------- .../admin/migrations/add_selectors.py | 4 +-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index e75e5654..cd0ccf26 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -538,6 +538,10 @@ def apply_moonworm_tasks( if "abi_method_hash" in tag ] + existing_selectors = [ + tag.split(":")[-1] for tag in chain(*existing_tags) if "abi_selector" in tag + ] + abi_hashes_dict = { hashlib.md5(json.dumps(method).encode("utf-8")).hexdigest(): method for method in abi @@ -545,17 +549,20 @@ def apply_moonworm_tasks( and (method.get("stateMutability", "") != "view") } + abi_selectors_dict = { + Web3.keccak( + text=abi_hashes_dict[hash]["name"] + + "(" + + ",".join(map(lambda x: x["type"], abi_hashes_dict[hash]["inputs"])) + + ")" + )[:4].hex(): method + for method in abi + if (method["type"] in ("event", "function")) + and (method.get("stateMutability", "") != "view") + } + for hash in abi_hashes_dict: if hash not in existing_hashes: - abi_selector = Web3.keccak( - text=abi_hashes_dict[hash]["name"] - + "(" - + ",".join( - map(lambda x: x["type"], abi_hashes_dict[hash]["inputs"]) - ) - + ")" - )[:4].hex() - moonworm_abi_tasks_entries_pack.append( { "title": address, diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index de0b96ea..daa79d90 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -51,7 +51,7 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: logger.warn(f"Unable to parse abi from task: {task.id}") continue - if not any([tag.startswith("selector:") for tag in task.tags]): + if not any([tag.startswith("abi_selector:") for tag in task.tags]): ## generate selector abi_selector = Web3.keccak( @@ -61,7 +61,7 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: + ")" )[:4].hex() - tags.append(f"#selector:{abi_selector}") + tags.append(f"abi_selector:{abi_selector}") entries_tags.append( { From 827823ab7a32d9ec545531d071f733101048cddc Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 19 Sep 2023 12:48:44 +0300 Subject: [PATCH 4/9] Add changes. --- moonstreamapi/moonstreamapi/actions.py | 35 ++++++++++---------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index cd0ccf26..bd832c51 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -526,34 +526,19 @@ def apply_moonworm_tasks( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - # create historical crawl task in journal - # will use create_entries_pack for creating entries in journal existing_tags = [entry.tags for entry in entries] - existing_hashes = [ - tag.split(":")[-1] - for tag in chain(*existing_tags) - if "abi_method_hash" in tag - ] - existing_selectors = [ tag.split(":")[-1] for tag in chain(*existing_tags) if "abi_selector" in tag ] - abi_hashes_dict = { - hashlib.md5(json.dumps(method).encode("utf-8")).hexdigest(): method - for method in abi - if (method["type"] in ("event", "function")) - and (method.get("stateMutability", "") != "view") - } - abi_selectors_dict = { Web3.keccak( - text=abi_hashes_dict[hash]["name"] + text=method["name"] + "(" - + ",".join(map(lambda x: x["type"], abi_hashes_dict[hash]["inputs"])) + + ",".join(map(lambda x: x["type"], method["inputs"])) + ")" )[:4].hex(): method for method in abi @@ -561,19 +546,25 @@ def apply_moonworm_tasks( and (method.get("stateMutability", "") != "view") } - for hash in abi_hashes_dict: - if hash not in existing_hashes: + for abi_selector in abi_selectors_dict: + if abi_selector not in existing_selectors: + hash = hashlib.md5( + json.dumps(abi_selectors_dict[abi_selector]).encode("utf-8") + ).hexdigest() + moonworm_abi_tasks_entries_pack.append( { "title": address, - "content": json.dumps(abi_hashes_dict[hash], indent=4), + "content": json.dumps( + abi_selectors_dict[abi_selector], indent=4 + ), "tags": [ f"address:{address}", - f"type:{abi_hashes_dict[hash]['type']}", + f"type:{abi_selectors_dict[abi_selector]['type']}", f"abi_method_hash:{hash}", f"abi_selector:{abi_selector}", f"subscription_type:{subscription_type}", - f"abi_name:{abi_hashes_dict[hash]['name']}", + f"abi_name:{abi_selectors_dict[abi_selector]['name']}", f"status:active", f"task_type:moonworm", f"moonworm_task_pickedup:False", # True if task picked up by moonworm-crawler(default each 120 sec) From 5a6c8f52194926ea27bb4f61d4511f5f3a3f4b3b Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 19 Sep 2023 13:40:24 +0300 Subject: [PATCH 5/9] Add changes. --- moonstreamapi/moonstreamapi/admin/cli.py | 9 +- .../admin/migrations/add_selectors.py | 98 ++++++++++++++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/moonstreamapi/moonstreamapi/admin/cli.py b/moonstreamapi/moonstreamapi/admin/cli.py index 84b14b1e..5a1b0f8b 100644 --- a/moonstreamapi/moonstreamapi/admin/cli.py +++ b/moonstreamapi/moonstreamapi/admin/cli.py @@ -12,7 +12,12 @@ from sqlalchemy.orm import with_expression from moonstreamdb.db import SessionLocal -from ..settings import BUGOUT_BROOD_URL, BUGOUT_SPIRE_URL, MOONSTREAM_APPLICATION_ID +from ..settings import ( + BUGOUT_BROOD_URL, + BUGOUT_SPIRE_URL, + MOONSTREAM_APPLICATION_ID, + MOONSTREAM_MOONWORM_TASKS_JOURNAL, +) from ..web3_provider import yield_web3_provider from . import subscription_types, subscriptions, moonworm_tasks, queries @@ -248,7 +253,7 @@ def moonworm_tasks_add_subscription_handler(args: argparse.Namespace) -> None: def main() -> None: cli_description = f"""Moonstream Admin CLI -Please m35ake sure that the following environment variables are set in your environment and exported to +Please make sure that the following environment variables are set in your environment and exported to subprocesses: 1. MOONSTREAM_APPLICATION_ID 2. MOONSTREAM_ADMIN_ACCESS_TOKEN diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index daa79d90..5c09accf 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -29,10 +29,13 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: moonworm_tasks = get_all_entries_from_search( journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - search_query="type:subscription !#version:2.0", - timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + search_query="#task_type:moonworm !#version:2.0", + limit=batch_size, + content=True, ) + logger.info(f"Found {len(moonworm_tasks)} moonworm tasks versions 1.0") + entries_tags = [] ## batch tasks @@ -41,14 +44,24 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: moonworm_tasks[i : i + batch_size] for i in range(0, len(moonworm_tasks), batch_size) ]: + count = 0 for task in task_batch: - tags = ["#version:2.0"] + tags = ["version:2.0"] ## get abi try: abi = json.loads(task.content) except Exception as e: - logger.warn(f"Unable to parse abi from task: {task.id}") + logger.warn( + f"Unable to parse abi from task: {task.entry_url.split()[-1]}: {e}" + ) + raise e + continue + + if "name" not in abi: + logger.warn( + f"Unable to find abi name in task: {task.entry_url.split()[-1]}" + ) continue if not any([tag.startswith("abi_selector:") for tag in task.tags]): @@ -63,6 +76,8 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: tags.append(f"abi_selector:{abi_selector}") + count += 1 + entries_tags.append( { "entry_id": task.entry_url.split("/")[-1], ## 😭 @@ -70,6 +85,8 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: } ) + logger.info(f"Found {count} missing selectors in batch {len(task_batch)} tasks") + ## update entries try: @@ -82,3 +99,76 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: except BugoutResponseException as e: logger.error(f"Unable to update entries tags: {e}") continue + + +def deduplicate_moonworm_task_by_selector(): + """ + Find moonworm tasks with same selector and remove old versions + """ + + moonworm_tasks = get_all_entries_from_search( + journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + search_query="#task_type:moonworm #version:2.0", + limit=100, + content=False, + ) + + logger.info(f"Found {len(moonworm_tasks)} moonworm tasks versions 2.0") + + ## loop over tasks + + selectors = {} + + for task in moonworm_tasks: + tags = task.tags + + ## get selector + selector = [tag for tag in tags if tag.startswith("abi_selector:")] + + if len(selector) == 0: + logger.warn( + f"Unable to find selector in task: {task.entry_url.split()[-1]}" + ) + continue + + selector = selector[0] + + if selector not in selectors: + selectors[selector] = {"entries": {}} + + selectors[selector]["entries"][task.entry_url.split("/")[-1]] = task.created_at + + logger.info(f"Found {len(selectors)} selectors") + + for selector, tasks_dict in selectors.items(): + if len(tasks_dict["entries"]) == 1: + continue + + ## find latest task + + latest_task_id = max( + tasks_dict["entries"], key=lambda key: tasks_dict["entries"][key] + ) + + ## remove all tasks except latest + + logger.info( + f"Found {len(tasks_dict['entries'])} tasks with selector {selector}" + ) + + for task_id in tasks_dict["entries"]: + if task_id == latest_task_id: + continue + + try: + bc.delete_entry( + journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, + entry_id=task_id, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + ) + except BugoutResponseException as e: + logger.error(f"Unable to delete entry: {e}") + continue + + logger.info(f"Deleted entry: {task_id}") From 1e848f72d4abb6061baf34585764ebeeaba01f7c Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 19 Sep 2023 13:44:54 +0300 Subject: [PATCH 6/9] Add deduplicates migration step. --- moonstreamapi/moonstreamapi/admin/cli.py | 9 ++++++++- .../moonstreamapi/admin/migrations/add_selectors.py | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/moonstreamapi/moonstreamapi/admin/cli.py b/moonstreamapi/moonstreamapi/admin/cli.py index 5a1b0f8b..12a96d4d 100644 --- a/moonstreamapi/moonstreamapi/admin/cli.py +++ b/moonstreamapi/moonstreamapi/admin/cli.py @@ -127,13 +127,20 @@ def migrations_run(args: argparse.Namespace) -> None: db_session = SessionLocal() try: if args.id == 20230904: - step_order = ["fill_missing_selectors_in_moonworm_tasks"] + step_order = [ + "fill_missing_selectors_in_moonworm_tasks", + "deduplicate_moonworm_tasks", + ] step_map: Dict[str, Dict[str, Any]] = { "upgrade": { "fill_missing_selectors_in_moonworm_tasks": { "action": add_selectors.fill_missing_selectors_in_moonworm_tasks, "description": "Get all moonworm jobs from moonworm journal and add selector tag if it not represent", }, + "deduplicate_moonworm_tasks": { + "action": add_selectors.deduplicate_moonworm_task_by_selector, + "description": "Deduplicate moonworm tasks by selector", + }, }, "downgrade": {}, } diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index 5c09accf..8ac0eefd 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -55,7 +55,6 @@ def fill_missing_selectors_in_moonworm_tasks() -> None: logger.warn( f"Unable to parse abi from task: {task.entry_url.split()[-1]}: {e}" ) - raise e continue if "name" not in abi: From 04c3c7aad556495ed30623b201656165ed1db021 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 19 Sep 2023 14:56:20 +0300 Subject: [PATCH 7/9] Add address split. --- .../admin/migrations/add_selectors.py | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index 8ac0eefd..909beb35 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -125,49 +125,60 @@ def deduplicate_moonworm_task_by_selector(): ## get selector selector = [tag for tag in tags if tag.startswith("abi_selector:")] + address = [tag for tag in tags if tag.startswith("address:")] + if len(selector) == 0: logger.warn( f"Unable to find selector in task: {task.entry_url.split()[-1]}" ) continue - selector = selector[0] + selector = selector[0].split(":")[1] - if selector not in selectors: - selectors[selector] = {"entries": {}} - - selectors[selector]["entries"][task.entry_url.split("/")[-1]] = task.created_at - - logger.info(f"Found {len(selectors)} selectors") - - for selector, tasks_dict in selectors.items(): - if len(tasks_dict["entries"]) == 1: + if len(address) == 0: + logger.warn(f"Unable to find address in task: {task.entry_url.split()[-1]}") continue - ## find latest task + address = address[0].split(":")[1] - latest_task_id = max( - tasks_dict["entries"], key=lambda key: tasks_dict["entries"][key] - ) + if address not in selectors: + selectors[address] = {selector: {"entries": {}}} - ## remove all tasks except latest + selectors[address][selector]["entries"][ + task.entry_url.split("/")[-1] + ] = task.created_at - logger.info( - f"Found {len(tasks_dict['entries'])} tasks with selector {selector}" - ) + logger.info(f"Found {len(selectors)} addresses") - for task_id in tasks_dict["entries"]: - if task_id == latest_task_id: + for address, selectors_dict in selectors.items(): + for selector, tasks_dict in selectors_dict.items(): + if len(tasks_dict["entries"]) == 1: continue - try: - bc.delete_entry( - journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, - entry_id=task_id, - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - ) - except BugoutResponseException as e: - logger.error(f"Unable to delete entry: {e}") - continue + ## find earliest task - logger.info(f"Deleted entry: {task_id}") + earliest_task_id = min( + tasks_dict["entries"], key=lambda key: tasks_dict["entries"][key] + ) + + ## remove all tasks except latest + + logger.info( + f"Found {len(tasks_dict['entries'])} tasks with selector {selector}" + ) + + for task_id in tasks_dict["entries"]: + if task_id == earliest_task_id: + continue + + try: + bc.delete_entry( + journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL, + entry_id=task_id, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + ) + except BugoutResponseException as e: + logger.error(f"Unable to delete entry: {e}") + continue + + logger.info(f"Deleted entry: {task_id}") From ac235cd15d6ce9918639d18127af38228d550f51 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 19 Sep 2023 15:10:55 +0300 Subject: [PATCH 8/9] Add initial selector key. --- .../moonstreamapi/admin/migrations/add_selectors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index 909beb35..f003aa29 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -142,7 +142,10 @@ def deduplicate_moonworm_task_by_selector(): address = address[0].split(":")[1] if address not in selectors: - selectors[address] = {selector: {"entries": {}}} + selectors[address] = {} + + if selector not in selectors[address]: + selectors[address][selector] = {"entries": {}} selectors[address][selector]["entries"][ task.entry_url.split("/")[-1] From 1ab1d022cc10ccbae8c13be4b0a434841f6bb1b4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 21 Sep 2023 14:23:13 +0300 Subject: [PATCH 9/9] Add changes. --- moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py index f003aa29..7da444af 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/add_selectors.py @@ -167,7 +167,7 @@ def deduplicate_moonworm_task_by_selector(): ## remove all tasks except latest logger.info( - f"Found {len(tasks_dict['entries'])} tasks with selector {selector}" + f"Found {len(tasks_dict['entries'])} tasks with selector {selector} erliest task {earliest_task_id} with created_at: {tasks_dict['entries'][earliest_task_id]}" ) for task_id in tasks_dict["entries"]: @@ -181,7 +181,7 @@ def deduplicate_moonworm_task_by_selector(): token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) except BugoutResponseException as e: - logger.error(f"Unable to delete entry: {e}") + logger.error(f"Unable to delete entry with id {task_id} : {e}") continue logger.info(f"Deleted entry: {task_id}")