From 77ee44884eaa4e08d8aa65128f0dfc718ef5bb02 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 15 Aug 2023 17:27:26 +0300 Subject: [PATCH 1/8] Add changes. --- moonstreamapi/moonstreamapi/routes/queries.py | 191 +++++++++--------- 1 file changed, 100 insertions(+), 91 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index e0f5f035..967db5aa 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -14,7 +14,7 @@ from bugout.data import ( BugoutSearchResult, ) from bugout.exceptions import BugoutResponseException -from fastapi import APIRouter, Body, Path, Request +from fastapi import APIRouter, Body, Path, Request, Query from moonstreamdb.blockchain import AvailableBlockchainType from sqlalchemy import text @@ -48,6 +48,10 @@ router = APIRouter( @router.get("/list", tags=["queries"]) async def get_list_of_queries_handler(request: Request) -> List[Dict[str, Any]]: + """ + Return list of queries which user own + """ + token = request.state.token # Check already existed queries @@ -73,7 +77,7 @@ async def create_query_handler( request: Request, query_applied: data.PreapprovedQuery = Body(...) ) -> BugoutJournalEntry: """ - Create query in bugout journal + Create query in bugout journal with preapprove status required approval from moonstream team """ token = request.state.token @@ -117,6 +121,7 @@ async def create_query_handler( title=f"Query:{query_name}", tags=["type:query"], content=query_applied.query, + timeout=10, ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) @@ -161,10 +166,15 @@ async def create_query_handler( @router.get("/templates", tags=["queries"]) def get_suggested_queries( - supported_interfaces: Optional[List[str]] = None, - address: Optional[str] = None, - title: Optional[str] = None, - limit: int = 10, + supported_interfaces: Optional[List[str]] = Query( + None, description="Supported interfaces in format: d73f4e3a erc1155" + ), + address: Optional[str] = Query( + None, + description="Query address for search if template applied to particular address", + ), + title: Optional[str] = Query(None, description="Query title for search"), + limit: int = Query(10), ) -> data.SuggestedQueriesResponse: """ Return set of suggested queries for user @@ -194,6 +204,7 @@ def get_suggested_queries( timeout=5, ) except BugoutResponseException as e: + logger.error(f"Error in get suggested queries templates: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -222,7 +233,7 @@ def get_suggested_queries( @router.get("/{query_name}/query", tags=["queries"]) async def get_query_handler( - request: Request, query_name: str + request: Request, query_name: str = Path(..., description="Query name") ) -> data.QueryInfoResponse: token = request.state.token @@ -248,7 +259,7 @@ async def get_query_handler( limit=1, ) except BugoutResponseException as e: - logger.error(f"Error in get query: {str(e)}") + logger.error(f"Error in search template: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -263,12 +274,10 @@ async def get_query_handler( ) try: - entries = bc.search( + entry = bc.get_entry( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", - limit=1, - timeout=5, + entry_id=query_id, ) except BugoutResponseException as e: logger.error(f"Error in get query: {str(e)}") @@ -276,23 +285,23 @@ async def get_query_handler( except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) - if len(entries.results) == 0: - raise MoonstreamHTTPException( - status_code=403, detail="Query not approved yet." - ) else: entries_results = cast(List[BugoutSearchResult], entries.results) query_id = entries_results[0].entry_url.split("/")[-1] + entry = entries_results[0] - entries_results = cast(List[BugoutSearchResult], entries.results) - entry = entries_results[0] + content = entry.content + tags = entry.tags + created_at = entry.created_at + updated_at = entry.updated_at + + if content is None: + raise MoonstreamHTTPException( + status_code=403, detail=f"Query is empty. Please update it." + ) try: - if entry.content is None: - raise MoonstreamHTTPException( - status_code=403, detail=f"Query is empty. Please update it." - ) - query = text(entry.content) + query = text(content) except Exception as e: raise MoonstreamHTTPException( status_code=500, internal_error=e, detail="Error in query parsing" @@ -301,8 +310,7 @@ async def get_query_handler( query_parameters_names = list(query._bindparams.keys()) tags_dict = { - tag.split(":")[0]: (tag.split(":")[1] if ":" in tag else True) - for tag in entry.tags + tag.split(":")[0]: (tag.split(":")[1] if ":" in tag else True) for tag in tags } query_parameters: Dict[str, Any] = {} @@ -313,23 +321,21 @@ async def get_query_handler( else: query_parameters[param] = None - print(type(entry.created_at)) - return data.QueryInfoResponse( - query=entry.content, + query=content, query_id=str(query_id), preapprove="preapprove" in tags_dict, approved="approved" in tags_dict, parameters=query_parameters, - created_at=entry.created_at, # type: ignore - updated_at=entry.updated_at, # type: ignore + created_at=created_at, # type: ignore + updated_at=updated_at, # type: ignore ) @router.put("/{query_name}", tags=["queries"]) async def update_query_handler( request: Request, - query_name: str, + query_name: str = Path(..., description="Query name"), request_update: data.UpdateQueryRequest = Body(...), ) -> BugoutJournalEntryContent: token = request.state.token @@ -367,9 +373,9 @@ async def update_query_handler( ) async def update_query_data_handler( request: Request, - query_name: str, + query_name: str = Path(..., description="Query name"), request_update: data.UpdateDataRequest = Body(...), -) -> Optional[data.QueryPresignUrl]: +) -> data.QueryPresignUrl: """ Request update data on S3 bucket """ @@ -407,7 +413,7 @@ async def update_query_data_handler( limit=1, ) except BugoutResponseException as e: - logger.error(f"Error in get query: {str(e)}") + logger.error(f"Error in search template: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -422,52 +428,55 @@ async def update_query_data_handler( ) try: - entries = bc.search( + entry = bc.get_entry( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", - limit=1, - timeout=5, + entry_id=query_id, ) + except BugoutResponseException as e: logger.error(f"Error in get query: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) - if len(entries.results) == 0: + ### check tags + + if "preapprove" in entry.tags or "approved" not in entry.tags: raise MoonstreamHTTPException( status_code=403, detail="Query not approved yet." ) + + content = entry.content + tags = entry.tags + else: entries_results = cast(List[BugoutSearchResult], entries.results) query_id = entries_results[0].entry_url.split("/")[-1] - - s3_response = None - - entries_results = cast(List[BugoutSearchResult], entries.results) - if entries_results[0].content: content = entries_results[0].content - tags = entries_results[0].tags + if content: file_type = "json" if "ext:csv" in tags: file_type = "csv" - - responce = requests.post( - f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/{query_id}/query_update", - json={ - "query": content, - "params": request_update.params, - "file_type": file_type, - "blockchain": request_update.blockchain - if request_update.blockchain - else None, - }, - timeout=5, - ) + try: + responce = requests.post( + f"{MOONSTREAM_CRAWLERS_SERVER_URL}:{MOONSTREAM_CRAWLERS_SERVER_PORT}/jobs/{query_id}/query_update", + json={ + "query": content, + "params": request_update.params, + "file_type": file_type, + "blockchain": request_update.blockchain + if request_update.blockchain + else None, + }, + timeout=5, + ) + except Exception as e: + logger.error(f"Error interaction with crawlers: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) if responce.status_code != 200: raise MoonstreamHTTPException( @@ -476,6 +485,10 @@ async def update_query_data_handler( ) s3_response = data.QueryPresignUrl(**responce.json()) + else: + raise MoonstreamHTTPException( + status_code=403, detail=f"Query is empty. Please update it." + ) return s3_response @@ -483,7 +496,7 @@ async def update_query_data_handler( @router.post("/{query_name}", tags=["queries"]) async def get_access_link_handler( request: Request, - query_name: str, + query_name: str = Path(..., description="Query name"), request_update: data.UpdateDataRequest = Body(...), ) -> Optional[data.QueryPresignUrl]: """ @@ -513,7 +526,7 @@ async def get_access_link_handler( limit=1, ) except BugoutResponseException as e: - logger.error(f"Error in get query: {str(e)}") + logger.error(f"Error in search template: {str(e)}") raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -528,12 +541,10 @@ async def get_access_link_handler( ) try: - entries = bc.search( + entry = bc.get_entry( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, - query=f"tag:approved tag:query_id:{query_id} !tag:preapprove", - limit=1, - timeout=5, + entry_id=query_id, ) except BugoutResponseException as e: logger.error(f"Error in get query: {str(e)}") @@ -541,38 +552,37 @@ async def get_access_link_handler( except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) - if len(entries.results) == 0: - raise MoonstreamHTTPException( - status_code=403, detail="Query not approved yet." - ) + else: + entry = cast(BugoutJournalEntry, entries.results[0]) - entries_results = cast(List[BugoutSearchResult], entries.results) + content = entry.content + tags = entry.tags + + if not content: + raise MoonstreamHTTPException( + status_code=403, detail=f"Query is empty. Please update it." + ) try: - s3_response = None + passed_params = dict(request_update.params) - if entries_results[0].content: - passed_params = dict(request_update.params) + file_type = "json" - tags = entries_results[0].tags + if "ext:csv" in tags: + file_type = "csv" - file_type = "json" + params_hash = query_parameter_hash(passed_params) - if "ext:csv" in tags: - file_type = "csv" + bucket = MOONSTREAM_S3_QUERIES_BUCKET + key = f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/{params_hash}/data.{file_type}" - params_hash = query_parameter_hash(passed_params) - - bucket = MOONSTREAM_S3_QUERIES_BUCKET - key = f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/{params_hash}/data.{file_type}" - - stats_presigned_url = generate_s3_access_links( - method_name="get_object", - bucket=bucket, - key=key, - expiration=300000, - http_method="GET", - ) - s3_response = data.QueryPresignUrl(url=stats_presigned_url) + stats_presigned_url = generate_s3_access_links( + method_name="get_object", + bucket=bucket, + key=key, + expiration=300000, + http_method="GET", + ) + s3_response = data.QueryPresignUrl(url=stats_presigned_url) except Exception as e: logger.error(f"Error in get access link: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -582,8 +592,7 @@ async def get_access_link_handler( @router.delete("/{query_name}", tags=["queries"]) async def remove_query_handler( - request: Request, - query_name: str, + request: Request, query_name: str = Path(..., description="Query name") ) -> BugoutJournalEntry: """ Request delete query from journal From cebd0cb777c8ffb0499269033b399304713734b4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 15 Aug 2023 17:30:25 +0300 Subject: [PATCH 2/8] Remove not rlated changes. --- engineapi/engineapi/routes/leaderboard.py | 89 +++++------------------ 1 file changed, 17 insertions(+), 72 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 4a4c8932..d5d95a73 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -6,7 +6,7 @@ import logging from uuid import UUID from web3 import Web3 -from fastapi import FastAPI, Request, Depends, Response, Query, Path, Body, Header +from fastapi import FastAPI, Request, Depends, Response, Query, Path, Body from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from typing import Any, Dict, List, Optional @@ -68,16 +68,13 @@ app.add_middleware( ) -@app.get("", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) -@app.get("/", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) +@app.get("", response_model=List[data.LeaderboardPosition]) +@app.get("/", response_model=List[data.LeaderboardPosition]) async def leaderboard( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> List[data.LeaderboardPosition]: """ Returns the leaderboard positions. @@ -111,19 +108,12 @@ async def leaderboard( return result -@app.post( - "", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] -) -@app.post( - "/", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] -) +@app.post("", response_model=data.LeaderboardCreatedResponse) +@app.post("/", response_model=data.LeaderboardCreatedResponse) async def create_leaderboard( request: Request, leaderboard: data.LeaderboardCreateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> data.LeaderboardCreatedResponse: """ @@ -162,19 +152,12 @@ async def create_leaderboard( ) -@app.put( - "/{leaderboard_id}", - response_model=data.LeaderboardUpdatedResponse, - tags=["Authorized Endpoints"], -) +@app.put("/{leaderboard_id}", response_model=data.LeaderboardUpdatedResponse) async def update_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), leaderboard: data.LeaderboardUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> data.LeaderboardUpdatedResponse: """ Update leaderboard. @@ -226,18 +209,11 @@ async def update_leaderboard( ) -@app.delete( - "/{leaderboard_id}", - response_model=data.LeaderboardDeletedResponse, - tags=["Authorized Endpoints"], -) +@app.delete("/{leaderboard_id}", response_model=data.LeaderboardDeletedResponse) async def delete_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> data.LeaderboardDeletedResponse: """ Delete leaderboard. @@ -287,17 +263,9 @@ async def delete_leaderboard( ) -@app.get( - "/leaderboards", - response_model=List[data.Leaderboard], - tags=["Authorized Endpoints"], -) +@app.get("/leaderboards", response_model=List[data.Leaderboard]) async def get_leaderboards( - request: Request, - db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + request: Request, db_session: Session = Depends(db.yield_db_session) ) -> List[data.Leaderboard]: """ Returns leaderboard list to which user has access. @@ -331,11 +299,7 @@ async def get_leaderboards( return results -@app.get( - "/count/addresses", - response_model=data.CountAddressesResponse, - tags=["Public Endpoints"], -) +@app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -361,9 +325,7 @@ async def count_addresses( return data.CountAddressesResponse(count=count) -@app.get( - "/info", response_model=data.LeaderboardInfoResponse, tags=["Public Endpoints"] -) +@app.get("/info", response_model=data.LeaderboardInfoResponse) async def leadeboard_info( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -391,11 +353,7 @@ async def leadeboard_info( ) -@app.get( - "/scores/changes", - response_model=List[data.LeaderboardScoresChangesResponse], - tags=["Public Endpoints"], -) +@app.get("/scores/changes") async def get_scores_changes( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -422,7 +380,7 @@ async def get_scores_changes( ] -@app.get("/quartiles", response_model=data.QuartilesResponse, tags=["Public Endpoints"]) +@app.get("/quartiles", response_model=data.QuartilesResponse) async def quartiles( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -458,11 +416,7 @@ async def quartiles( ) -@app.get( - "/position", - response_model=List[data.LeaderboardPosition], - tags=["Public Endpoints"], -) +@app.get("/position", response_model=List[data.LeaderboardPosition]) async def position( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), address: str = Query(..., description="Address to get position for."), @@ -511,9 +465,7 @@ async def position( return results -@app.get( - "/rank", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"] -) +@app.get("/rank", response_model=List[data.LeaderboardPosition]) async def rank( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), rank: int = Query(1, description="Rank to get."), @@ -552,7 +504,7 @@ async def rank( return results -@app.get("/ranks", response_model=List[data.RanksResponse], tags=["Public Endpoints"]) +@app.get("/ranks", response_model=List[data.RanksResponse]) async def ranks( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -585,11 +537,7 @@ async def ranks( return results -@app.put( - "/{leaderboard_id}/scores", - response_model=List[data.LeaderboardScore], - tags=["Authorized Endpoints"], -) +@app.put("/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore]) async def leaderboard_push_scores( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), @@ -604,9 +552,6 @@ async def leaderboard_push_scores( True, description="Normalize addresses to checksum." ), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> List[data.LeaderboardScore]: """ Put the leaderboard to the database. From 697de0bcc17dd07967c7317888094b8068d5881e Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 18 Sep 2023 04:21:16 +0300 Subject: [PATCH 3/8] Add changes. --- engineapi/engineapi/routes/leaderboard.py | 66 +++++++++++++++++------ 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 3b216305..e1d45014 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -86,8 +86,8 @@ app.add_middleware( ) -@app.get("", response_model=List[data.LeaderboardPosition]) -@app.get("/", response_model=List[data.LeaderboardPosition]) +@app.get("", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) +@app.get("/", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) async def leaderboard( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), limit: int = Query(10), @@ -126,8 +126,12 @@ async def leaderboard( return result -@app.post("", response_model=data.LeaderboardCreatedResponse) -@app.post("/", response_model=data.LeaderboardCreatedResponse) +@app.post( + "", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] +) +@app.post( + "/", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] +) async def create_leaderboard( request: Request, leaderboard: data.LeaderboardCreateRequest = Body(...), @@ -171,7 +175,11 @@ async def create_leaderboard( ) -@app.put("/{leaderboard_id}", response_model=data.LeaderboardUpdatedResponse) +@app.put( + "/{leaderboard_id}", + response_model=data.LeaderboardUpdatedResponse, + tags=["Authorized Endpoints"], +) async def update_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), @@ -229,7 +237,11 @@ async def update_leaderboard( ) -@app.delete("/{leaderboard_id}", response_model=data.LeaderboardDeletedResponse) +@app.delete( + "/{leaderboard_id}", + response_model=data.LeaderboardDeletedResponse, + tags=["Authorized Endpoints"], +) async def delete_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), @@ -284,7 +296,11 @@ async def delete_leaderboard( ) -@app.get("/leaderboards", response_model=List[data.Leaderboard]) +@app.get( + "/leaderboards", + response_model=List[data.Leaderboard], + tags=["Authorized Endpoints"], +) async def get_leaderboards( request: Request, db_session: Session = Depends(db.yield_db_session), @@ -322,7 +338,11 @@ async def get_leaderboards( return results -@app.get("/count/addresses", response_model=data.CountAddressesResponse) +@app.get( + "/count/addresses", + response_model=data.CountAddressesResponse, + tags=["Public Endpoints"], +) async def count_addresses( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -348,7 +368,9 @@ async def count_addresses( return data.CountAddressesResponse(count=count) -@app.get("/info", response_model=data.LeaderboardInfoResponse) +@app.get( + "/info", response_model=data.LeaderboardInfoResponse, tags=["Public Endpoints"] +) async def leadeboard_info( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -376,7 +398,11 @@ async def leadeboard_info( ) -@app.get("/scores/changes") +@app.get( + "/scores/changes", + response_model=List[data.LeaderboardScoresChangesResponse], + tags=["Public Endpoints"], +) async def get_scores_changes( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -403,7 +429,7 @@ async def get_scores_changes( ] -@app.get("/quartiles", response_model=data.QuartilesResponse) +@app.get("/quartiles", response_model=data.QuartilesResponse, tags=["Public Endpoints"]) async def quartiles( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -439,7 +465,11 @@ async def quartiles( ) -@app.get("/position", response_model=List[data.LeaderboardPosition]) +@app.get( + "/position", + response_model=List[data.LeaderboardPosition], + tags=["Public Endpoints"], +) async def position( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), address: str = Query(..., description="Address to get position for."), @@ -488,7 +518,9 @@ async def position( return results -@app.get("/rank", response_model=List[data.LeaderboardPosition]) +@app.get( + "/rank", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"] +) async def rank( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), rank: int = Query(1, description="Rank to get."), @@ -527,7 +559,7 @@ async def rank( return results -@app.get("/ranks", response_model=List[data.RanksResponse]) +@app.get("/ranks", response_model=List[data.RanksResponse], tags=["Public Endpoints"]) async def ranks( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -560,7 +592,11 @@ async def ranks( return results -@app.put("/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore]) +@app.put( + "/{leaderboard_id}/scores", + response_model=List[data.LeaderboardScore], + tags=["Authorized Endpoints"], +) async def leaderboard_push_scores( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), From 0c7839cd9185bef406287b22056cf868c6562261 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 21 Sep 2023 14:34:43 +0300 Subject: [PATCH 4/8] Add changes. --- moonstreamapi/moonstreamapi/routes/queries.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 967db5aa..5a1a0189 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -36,6 +36,7 @@ from ..settings import ( MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE, MOONSTREAM_S3_QUERIES_BUCKET, MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, + BUGOUT_REQUEST_TIMEOUT_SECONDS, ) from ..settings import bugout_client as bc @@ -121,7 +122,7 @@ async def create_query_handler( title=f"Query:{query_name}", tags=["type:query"], content=query_applied.query, - timeout=10, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS * 2, ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) @@ -201,7 +202,7 @@ def get_suggested_queries( journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, query=query, limit=limit, - timeout=5, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) except BugoutResponseException as e: logger.error(f"Error in get suggested queries templates: {str(e)}") From 105f29dfbea94e85eabf8307d632e923739e62ba Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 18 Oct 2023 07:48:45 +0300 Subject: [PATCH 5/8] Reduce max blocks batch. --- .../mooncrawl/moonworm_crawler/event_crawler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py index a2c5f741..73f14f1b 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py @@ -158,12 +158,13 @@ def _autoscale_crawl_events( all_events = [] for job in jobs: raw_events, batch_size = moonworm_autoscale_crawl_events( - web3, - job.event_abi, - from_block, - to_block, - batch_size, - job.contracts[0], + web3=web3, + event_abi=job.event_abi, + from_block=from_block, + to_block=to_block, + batch_size=batch_size, + contract_address=job.contracts[0], + max_blocks_batch=3000, ) for raw_event in raw_events: raw_event["blockTimestamp"] = get_block_timestamp( From 8e9023afc5afa82b9efeb0b4610d291274ec0920 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 18 Oct 2023 11:59:47 +0000 Subject: [PATCH 6/8] JSON RPC request ID now support uint64, string and null --- nodebalancer/cmd/nodebalancer/blockchain.go | 37 ++++++++++----------- nodebalancer/cmd/nodebalancer/middleware.go | 12 +++++++ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/nodebalancer/cmd/nodebalancer/blockchain.go b/nodebalancer/cmd/nodebalancer/blockchain.go index b0d8c57e..82598995 100644 --- a/nodebalancer/cmd/nodebalancer/blockchain.go +++ b/nodebalancer/cmd/nodebalancer/blockchain.go @@ -38,25 +38,24 @@ var ( "web3_clientVersion": true, // zksync methods - "zks_estimateFee": true, - "zks_estimateGasL1ToL2": true, - "zks_getAllAccountBalances": true, - "zks_getBlockDetails": true, - "zks_getBridgeContracts": true, - "zks_getBytecodeByHash": true, - "zks_getConfirmedTokens": true, - "zks_getL1BatchBlockRange": true, - "zks_getL1BatchDetails": true, - "zks_getL2ToL1LogProof": true, - "zks_getL2ToL1MsgProof": true, - "zks_getMainContract": true, + "zks_estimateFee": true, + "zks_estimateGasL1ToL2": true, + "zks_getAllAccountBalances": true, + "zks_getBlockDetails": true, + "zks_getBridgeContracts": true, + "zks_getBytecodeByHash": true, + "zks_getConfirmedTokens": true, + "zks_getL1BatchBlockRange": true, + "zks_getL1BatchDetails": true, + "zks_getL2ToL1LogProof": true, + "zks_getL2ToL1MsgProof": true, + "zks_getMainContract": true, "zks_getRawBlockTransactions": true, - "zks_getTestnetPaymaster": true, - "zks_getTokenPrice": true, - "zks_getTransactionDetails": true, - "zks_L1BatchNumber": true, - "zks_L1ChainId": true, - + "zks_getTestnetPaymaster": true, + "zks_getTokenPrice": true, + "zks_getTransactionDetails": true, + "zks_L1BatchNumber": true, + "zks_L1ChainId": true, } ) @@ -64,7 +63,7 @@ type JSONRPCRequest struct { Jsonrpc string `json:"jsonrpc"` Method string `json:"method"` Params []interface{} `json:"params"` - ID uint64 `json:"id"` + ID interface{} `json:"id"` // According to the JSON-RPC specification, the id can be a string, number, or null } type BlockchainConfig struct { diff --git a/nodebalancer/cmd/nodebalancer/middleware.go b/nodebalancer/cmd/nodebalancer/middleware.go index e5068909..e99ec037 100644 --- a/nodebalancer/cmd/nodebalancer/middleware.go +++ b/nodebalancer/cmd/nodebalancer/middleware.go @@ -390,6 +390,7 @@ func jsonrpcRequestParser(body []byte) ([]JSONRPCRequest, error) { var jsonrpcRequest []JSONRPCRequest firstByte := bytes.TrimLeft(body, " \t\r\n") + switch { case len(firstByte) > 0 && firstByte[0] == '[': err := json.Unmarshal(body, &jsonrpcRequest) @@ -407,6 +408,17 @@ func jsonrpcRequestParser(body []byte) ([]JSONRPCRequest, error) { return nil, fmt.Errorf("incorrect first byte in JSON RPC request") } + for _, req := range jsonrpcRequest { + switch v := req.ID.(type) { + case float64: + req.ID = uint64(v) + case string: + case nil: + default: + return nil, fmt.Errorf("unexpected type for id: %T", v) + } + } + return jsonrpcRequest, nil } From 1439fb10fa61ff48a8aaaf6608453422eb29a489 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 18 Oct 2023 13:21:13 +0000 Subject: [PATCH 7/8] Path losing url during revers proxy fix --- nodebalancer/cmd/nodebalancer/server.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nodebalancer/cmd/nodebalancer/server.go b/nodebalancer/cmd/nodebalancer/server.go index c3c4c0fd..648fa3da 100644 --- a/nodebalancer/cmd/nodebalancer/server.go +++ b/nodebalancer/cmd/nodebalancer/server.go @@ -201,6 +201,11 @@ func Server() { r.URL.RawQuery = "" r.Header.Del(strings.Title(NB_ACCESS_ID_HEADER)) r.Header.Del(strings.Title(NB_DATA_SOURCE_HEADER)) + + r.URL.Scheme = endpoint.Scheme + r.URL.Host = endpoint.Host + r.URL.Path = endpoint.Path + // Change r.Host from nodebalancer's to end host so TLS check will be passed r.Host = r.URL.Host } From 01eeb2f0e6aa2d23a1ef11fdabeb4bd2983246c3 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 24 Oct 2023 13:15:03 +0300 Subject: [PATCH 8/8] Bump version. --- moonstreamapi/moonstreamapi/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/version.py b/moonstreamapi/moonstreamapi/version.py index 0928d039..5558da62 100644 --- a/moonstreamapi/moonstreamapi/version.py +++ b/moonstreamapi/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.3.0" +MOONSTREAMAPI_VERSION = "0.3.1"