From d07ec13bf66c55a841dc46d15430e838e0c45a8b Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 11 Sep 2023 16:53:14 +0300 Subject: [PATCH 1/3] Add Shadowcorns. --- .../state_crawler/jobs/polygon-jobs.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json index 43ff1fe1..3f230278 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json @@ -188,5 +188,36 @@ } ], "address": "0x44b3f42e2BF34F62868Ff9e9dAb7C2F807ba97Cb" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "polygon", + "params": { + "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" + }, + "keys": [ + "token_id" + ] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xa7D50EE3D7485288107664cf758E877a0D351725" } ] \ No newline at end of file From bb4bc7f778808df95b7498c8e4003d3453270a95 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 11 Sep 2023 17:27:03 +0300 Subject: [PATCH 2/3] Add improvments. --- crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py | 5 ++++- crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py index 558f9915..94497348 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py @@ -55,7 +55,10 @@ def crawl_uri(metadata_uri: str) -> Any: try: response = urllib.request.urlopen(metadata_uri, timeout=10) - if response.status == 200: + if ( + metadata_uri.startswith("data:application/json") + or response.status == 200 + ): result = json.loads(response.read()) break retry += 1 diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py index 4c3c771b..7cc2c3bc 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py @@ -132,6 +132,7 @@ def get_current_metadata_for_address( WHERE address = :address AND label = :label + AND label_data ->>'metadata' != 'null' ORDER BY label_data ->> 'token_id' ASC, block_number :: INT DESC; From 21a0fc415783367e45620a8d60542ab47bbb8170 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 11 Sep 2023 19:30:00 +0300 Subject: [PATCH 3/3] Fix lint.. --- .../mooncrawl/stats_worker/dashboard.py | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 25dc93cd..c33dc632 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -160,7 +160,7 @@ def generate_data( .filter(label_model.address == address) .filter(label_model.label == crawler_label) .filter(label_model.label_data["type"].astext == metric_type) - .filter(in_op(label_model.label_data["name"].astext, functions)) + .filter(in_op(label_model.label_data["name"].astext, functions)) # type: ignore .filter( label_model.block_timestamp >= sqlalchemy_cast( @@ -225,8 +225,8 @@ def generate_data( .join( with_empty_times_series_with_tags, and_( - with_empty_times_series_with_tags.c.label - == with_timetrashold_data.c.label, + with_empty_times_series_with_tags.c.label # type: ignore + == with_timetrashold_data.c.label, # type: ignore with_empty_times_series_with_tags.c.timeseries_points == with_timetrashold_data.c.timeseries_points, ), @@ -405,7 +405,7 @@ def process_external_merged( for extcall in external_calls_normalized: try: - contract = web3_client.eth.contract( + contract = web3_client.eth.contract( # type: ignore address=extcall["address"], abi=extcall["abi"] ) response = contract.functions[extcall["name"]]( @@ -472,7 +472,7 @@ def process_external( for extcall in external_calls: try: - contract = web3_client.eth.contract( + contract = web3_client.eth.contract( # type: ignore address=extcall["address"], abi=extcall["abi"] ) response = contract.functions[extcall["name"]]( @@ -709,25 +709,29 @@ def stats_generate_handler(args: argparse.Namespace): address = subscription.address if address not in address_dashboard_id_subscription_id_tree: - address_dashboard_id_subscription_id_tree[address] = {} + address_dashboard_id_subscription_id_tree[address] = {} # type: ignore if ( str(dashboard.id) not in address_dashboard_id_subscription_id_tree ): - address_dashboard_id_subscription_id_tree[address][ + address_dashboard_id_subscription_id_tree[address][ # type: ignore str(dashboard.id) ] = [] if ( subscription_id - not in address_dashboard_id_subscription_id_tree[ - address - ][str(dashboard.id)] - ): - address_dashboard_id_subscription_id_tree[address][ + not in address_dashboard_id_subscription_id_tree[ # type: ignore + address # type: ignore + ][ str(dashboard.id) - ].append(subscription_id) + ] + ): # type: ignore + address_dashboard_id_subscription_id_tree[address][ # type: ignore + str(dashboard.id) + ].append( + subscription_id + ) abi = None if "abi" in subscription.secondary_fields: @@ -761,23 +765,23 @@ def stats_generate_handler(args: argparse.Namespace): ) if address not in merged_events: - merged_events[address] = {} - merged_events[address]["merged"] = set() + merged_events[address] = {} # type: ignore + merged_events[address]["merged"] = set() # type: ignore if address not in merged_functions: - merged_functions[address] = {} - merged_functions[address]["merged"] = set() + merged_functions[address] = {} # type: ignore + merged_functions[address]["merged"] = set() # type: ignore - if str(dashboard.id) not in merged_events[address]: - merged_events[address][str(dashboard.id)] = {} + if str(dashboard.id) not in merged_events[address]: # type: ignore + merged_events[address][str(dashboard.id)] = {} # type: ignore - if str(dashboard.id) not in merged_functions[address]: - merged_functions[address][str(dashboard.id)] = {} + if str(dashboard.id) not in merged_functions[address]: # type: ignore + merged_functions[address][str(dashboard.id)] = {} # type: ignore - merged_events[address][str(dashboard.id)][ + merged_events[address][str(dashboard.id)][ # type: ignore subscription_id ] = events - merged_functions[address][str(dashboard.id)][ + merged_functions[address][str(dashboard.id)][ # type: ignore subscription_id ] = methods @@ -844,10 +848,10 @@ def stats_generate_handler(args: argparse.Namespace): # Fill merged events and functions calls for all subscriptions for event in events: - merged_events[address]["merged"].add(event) + merged_events[address]["merged"].add(event) # type: ignore for method in methods: - merged_functions[address]["merged"].add(method) + merged_functions[address]["merged"].add(method) # type: ignore except Exception as e: logger.error(f"Error while merging subscriptions: {e}") @@ -996,7 +1000,7 @@ def stats_generate_handler(args: argparse.Namespace): "statistics", f"blockchain:{args.blockchain}" f"subscriptions:{subscription_id}", - f"dashboard:{dashboard}", + f"dashboard:{dashboard}", # type: ignore ], ) logger.error(err) @@ -1040,7 +1044,7 @@ def stats_generate_api_task( subscription_id = dashboard_subscription_filters["subscription_id"] subscription_type_id = None - for required_field in subscription_by_id[ + for required_field in subscription_by_id[ # type: ignore subscription_id ].required_fields: if "subscription_type_id" in required_field: @@ -1065,8 +1069,8 @@ def stats_generate_api_task( crawler_label = CRAWLER_LABEL abi = None - if "abi" in subscription_by_id[subscription_id].secondary_fields: - abi = subscription_by_id[subscription_id].secondary_fields["abi"] + if "abi" in subscription_by_id[subscription_id].secondary_fields: # type: ignore + abi = subscription_by_id[subscription_id].secondary_fields["abi"] # type: ignore # Read required events, functions and web3_call form ABI if abi is None: @@ -1097,7 +1101,7 @@ def stats_generate_api_task( db_session=db_session, events=events, blockchain_type=blockchain_type, - address=address, + address=address, # type: ignore crawler_label=crawler_label, abi_json=abi_json, access_id=access_id, @@ -1127,7 +1131,7 @@ def stats_generate_api_task( functions_calls_data = generate_data( db_session=db_session, blockchain_type=blockchain_type, - address=address, + address=address, # type: ignore timescale=timescale, functions=methods, start=start_date, @@ -1140,7 +1144,7 @@ def stats_generate_api_task( events_data = generate_data( db_session=db_session, blockchain_type=blockchain_type, - address=address, + address=address, # type: ignore timescale=timescale, functions=events, start=start_date, @@ -1154,7 +1158,7 @@ def stats_generate_api_task( push_statistics( statistics_data=s3_data_object, subscription_type_id=subscription_type_id, - address=address, + address=address, # type: ignore timescale=timescale, bucket=MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, # type: ignore dashboard_id=dashboard.id, @@ -1165,7 +1169,7 @@ def stats_generate_api_task( [ "dashboard", "statistics", - f"subscriptions:{subscription_id}", + f"subscriptions:{subscription_id}", # type: ignore f"dashboard:{str(dashboard.id)}", ], )