Merge pull request #1045 from moonstream-to/usage-bugout-reports

Usage bugout reports

moonstreamapi/configs/sample.env

@@ -17,6 +17,7 @@ export MOONSTREAM_DATA_JOURNAL_ID="<bugout_journal_id_to_store_blockchain_data>"
 export HUMBUG_TXPOOL_CLIENT_ID="<Bugout_Humbug_client_id_for_txpool_transactions_in_journal>"
 export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
 export MOONSTREAM_QUERIES_JOURNAL_ID="<bugout_journal_id_where_store_queries_for_executing>"
+export MOONSTREAM_USAGE_REPORTS_JOURNAL_ID="<bugout_journal_id_where_save_generated_reports>"
 
 # Set following parameters if AWS node instance and S3 smartcontracts configured
 export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_to_store_smart_contracts>"
@@ -26,4 +27,4 @@ export MOONSTREAM_S3_QUERIES_BUCKET="<AWS_S3_bucket_to_store_sql_queries>"
 export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev"
 
 # Set the following variables in the most reasonable manner for your development environment
-export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout_umbug_token_for_crash_reports>"
+export HUMBUG_REPORTER_BACKEND_TOKEN="<Bugout_umbug_token_for_crash_reports>"

moonstreamapi/moonstreamapi/admin/cli.py

@@ -16,8 +16,10 @@ from moonstreamdb.db import SessionLocal
 from ..settings import (
     BUGOUT_BROOD_URL,
     BUGOUT_SPIRE_URL,
+    MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_APPLICATION_ID,
     MOONSTREAM_MOONWORM_TASKS_JOURNAL,
+    MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
 )
 from ..web3_provider import yield_web3_provider
 
@@ -265,6 +267,17 @@ def generate_usage_handler(args: argparse.Namespace) -> None:
         contracts=args.contracts,
     )
 
+    if MOONSTREAM_USAGE_REPORTS_JOURNAL_ID is not None:
+
+        usage.push_report_to_bugout_journal(
+            name=args.name,
+            user_id=args.user_id,
+            month=args.month,
+            journal_id=MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
+            report=usage_info,
+            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+        )
+
     if args.output is not None:
         # create path if not exists
 
@@ -273,8 +286,6 @@ def generate_usage_handler(args: argparse.Namespace) -> None:
 
         with open(args.output, "w") as output_file:
             output_file.write(json.dumps(usage_info, indent=4))
-    else:
-        logger.info(json.dumps(usage_info, indent=4))
 
 
 def main() -> None:
@@ -578,6 +589,14 @@ This CLI is configured to work with the following API URLs:
         type=str,
         help="User token for which to generate usage (not implemented yet - use user-token instead)",
     )
+
+    generate_usage_parser.add_argument(
+        "--name",
+        type=str,
+        help="Name of the user for which to generate usage",
+        default="",
+    )
+
     generate_usage_parser.add_argument(
         "--contracts",
         required=False,

moonstreamapi/moonstreamapi/admin/usage.py

@@ -5,7 +5,7 @@ import logging
 import time
 import uuid
 import requests  # type: ignore
-import os
+import textwrap
 
 from ..actions import get_all_entries_from_search
 from ..settings import bugout_client as bc
@@ -14,6 +14,7 @@ from ..settings import (
     BUGOUT_REQUEST_TIMEOUT_SECONDS,
     MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN,
     MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
+    MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
 )
 from ..data import BUGOUT_RESOURCE_QUERY_RESOLVER
 
@@ -101,13 +102,39 @@ def generate_leaderboard_owners(
     Get list of all leaderboard and add owners to it.
     """
 
+    leaderboards_cache = {}
+
     leaderboard_owners = []
 
+    ### Get leaderboard owners cache entry
+    entries = []
+
+    if MOONSTREAM_USAGE_REPORTS_JOURNAL_ID is not None:
+        try:
+            entries = get_all_entries_from_search(
+                journal_id=MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
+                search_query=f"tag:leaderboard_owners tag:cache",
+                limit=100,
+                token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+                content=True,
+            )
+        except Exception as e:
+            logger.error(f"Error getting leaderboard_owners_cache entry: {e}")
+
+    if len(entries) > 0:
+        try:
+            leaderboards_cache = json.loads(entries[0].content)
+        except Exception as e:
+            logger.error(f"Error loading leaderboard_owners_cache: {e}")
+
     for leaderboard in leaderboards:
-        # breakpoint()
         leaderboard_id = leaderboard.resource_data["leaderboard_id"]
         resource_id = leaderboard.id
 
+        if leaderboard_id in leaderboards_cache:
+            leaderboard_owners.append(leaderboards_cache[leaderboard_id])
+            continue
+
         holders: BugoutResourceHolders = bc.get_resource_holders(
             token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
             resource_id=resource_id,
@@ -136,6 +163,56 @@ def generate_leaderboard_owners(
             }
         )
 
+        leaderboards_cache[leaderboard_id] = {
+            "leaderboard_id": leaderboard_id,
+            "owner": str(owner),
+            "resource_id": str(resource_id),
+            "created_at": str(leaderboard.created_at),
+            "updated_at": str(leaderboard.updated_at),
+        }
+
+    ### update cache
+
+    if MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID is not None:
+
+        if len(entries) > 0:
+            try:
+                bc.update_entry_content(
+                    token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+                    journal_id=MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
+                    entry_id=entries[0].entry_url.split("/")[-1],
+                    content=textwrap.indent(
+                        json.dumps(leaderboards_cache, indent=4), "    "
+                    ),
+                    title=entries[0].title,
+                    timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+                )
+            except Exception as e:
+                logger.error(
+                    f"Error updating leaderboard_owners_cache entry: {e} continue..."
+                )
+        else:
+            title = "leaderboard_owners_cache"
+            tags = [
+                "leaderboard_owners",
+                "cache",
+            ]
+            try:
+                report_entry = bc.create_entry(
+                    token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+                    journal_id=MOONSTREAM_USAGE_REPORTS_JOURNAL_ID,
+                    title=title,
+                    content=textwrap.indent(
+                        json.dumps(leaderboards_cache, indent=4), "    "
+                    ),
+                    tags=tags,
+                    timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+                )
+            except Exception as e:
+                logger.error(
+                    f"Error creating leaderboard_owners_cache entry: {e} continue..."
+                )
+
     return leaderboard_owners
 
 
@@ -244,6 +321,9 @@ def collect_usage_information(
             "update_activated": True if "status:active" in tags else False,
         }
 
+    logger.info("Found leaderboards: %s", len(leaderboards))
+    logger.info("Fill leaderboards with users data")
+
     user_leaderboards = []
 
     for leaderboard in leaderboards:
@@ -316,3 +396,72 @@ def collect_usage_information(
         "leaderboards_amount": len(leaderboards),
         "contracts": contract_data,
     }
+
+
+def push_report_to_bugout_journal(
+    name: str,
+    user_id: str,
+    month: str,
+    journal_id: str,
+    report: Dict[str, Any],
+    token: str,
+) -> None:
+    """
+    Push report to bugout journal.
+    """
+    ### search by month if entry already exists
+
+    entries = get_all_entries_from_search(
+        journal_id=journal_id,
+        search_query=f"tag:month:{month} tag:report tag:user_id:{user_id}",
+        limit=100,
+        token=token,
+    )
+
+    if len(entries) > 0:
+        entry = entries[0]
+
+        ### ensure additional tags
+
+        tags = entry.tags
+
+        if f"customer:{name}" not in tags:
+            tags.append(f"customer:{name}")
+
+        if "moonstream" not in tags:
+            tags.append("moonstream")
+
+        bc.update_entry_content(
+            token=token,
+            journal_id=journal_id,
+            entry_id=entry.entry_url.split("/")[-1],
+            content=textwrap.indent(json.dumps(report, indent=4), "    "),
+            title=entry.title,
+            tags=tags,
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+        )
+
+        logger.info("Report entry updated: %s", entry.entry_url)
+
+    else:
+
+        title = f"{name} - {month} - {user_id}"
+
+        tags = [
+            f"month:{month}",
+            "report",
+            f"user_id:{user_id}",
+            "moonstream",
+            f"customer:{name}",
+        ]
+
+        report_entry = bc.create_entry(
+            token=token,
+            journal_id=journal_id,
+            title=title,
+            content=textwrap.indent(json.dumps(report, indent=4), "    "),
+            tags=tags,
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+        )
+
+        logger.info("Report entry created: %s", report_entry.id)

moonstreamapi/moonstreamapi/settings.py

@@ -320,3 +320,8 @@ if MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID == "":
     raise ValueError(
         "MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID environment variable must be set"
     )
+
+
+MOONSTREAM_USAGE_REPORTS_JOURNAL_ID = os.environ.get(
+    "MOONSTREAM_USAGE_REPORTS_JOURNAL_ID"
+)

moonstreamapi/moonstreamapi/version.py

@@ -2,4 +2,4 @@
 Moonstream library and API version.
 """
 
-MOONSTREAMAPI_VERSION = "0.3.4"
+MOONSTREAMAPI_VERSION = "0.3.5"