Add missing version.

pull/1011/head
Andrey 2024-01-29 11:42:07 +02:00
rodzic 38947e1379
commit b3aa979d78
2 zmienionych plików z 17 dodań i 12 usunięć

Wyświetl plik

@ -504,7 +504,7 @@ def get_all_entries_from_search(
limit=limit,
offset=offset,
)
results.extend(existing_methods.results) # type: ignore
results.extend(existing_methods.results) # type: ignore
return results
@ -573,6 +573,7 @@ def apply_moonworm_tasks(
f"moonworm_task_pickedup:False", # True if task picked up by moonworm-crawler(default each 120 sec)
f"historical_crawl_status:pending", # pending, in_progress, done
f"progress:0", # 0-100 %
f"version:2.0",
],
}
)

Wyświetl plik

@ -86,18 +86,22 @@ def fill_missing_selectors_in_moonworm_tasks() -> None:
logger.info(f"Found {count} missing selectors in batch {len(task_batch)} tasks")
## update entries
# ## update entries tags in batch
try:
bc.create_entries_tags(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
entries_tags=entries_tags,
timeout=15,
)
except BugoutResponseException as e:
logger.error(f"Unable to update entries tags: {e}")
continue
for batch in [
entries_tags[i : i + batch_size]
for i in range(0, len(entries_tags), batch_size)
]:
try:
bc.create_entries_tags(
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
entries_tags=batch,
timeout=15,
)
except BugoutResponseException as e:
logger.error(f"Unable to update entries tags: {e}")
continue
def deduplicate_moonworm_task_by_selector():