kopia lustrzana https://github.com/bugout-dev/moonstream
Add missing version.
rodzic
38947e1379
commit
b3aa979d78
|
@ -504,7 +504,7 @@ def get_all_entries_from_search(
|
|||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
results.extend(existing_methods.results) # type: ignore
|
||||
|
||||
return results
|
||||
|
||||
|
@ -573,6 +573,7 @@ def apply_moonworm_tasks(
|
|||
f"moonworm_task_pickedup:False", # True if task picked up by moonworm-crawler(default each 120 sec)
|
||||
f"historical_crawl_status:pending", # pending, in_progress, done
|
||||
f"progress:0", # 0-100 %
|
||||
f"version:2.0",
|
||||
],
|
||||
}
|
||||
)
|
||||
|
|
|
@ -86,18 +86,22 @@ def fill_missing_selectors_in_moonworm_tasks() -> None:
|
|||
|
||||
logger.info(f"Found {count} missing selectors in batch {len(task_batch)} tasks")
|
||||
|
||||
## update entries
|
||||
# ## update entries tags in batch
|
||||
|
||||
try:
|
||||
bc.create_entries_tags(
|
||||
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
entries_tags=entries_tags,
|
||||
timeout=15,
|
||||
)
|
||||
except BugoutResponseException as e:
|
||||
logger.error(f"Unable to update entries tags: {e}")
|
||||
continue
|
||||
for batch in [
|
||||
entries_tags[i : i + batch_size]
|
||||
for i in range(0, len(entries_tags), batch_size)
|
||||
]:
|
||||
try:
|
||||
bc.create_entries_tags(
|
||||
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
entries_tags=batch,
|
||||
timeout=15,
|
||||
)
|
||||
except BugoutResponseException as e:
|
||||
logger.error(f"Unable to update entries tags: {e}")
|
||||
continue
|
||||
|
||||
|
||||
def deduplicate_moonworm_task_by_selector():
|
||||
|
|
Ładowanie…
Reference in New Issue