diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py index 558f9915..94497348 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py @@ -55,7 +55,10 @@ def crawl_uri(metadata_uri: str) -> Any: try: response = urllib.request.urlopen(metadata_uri, timeout=10) - if response.status == 200: + if ( + metadata_uri.startswith("data:application/json") + or response.status == 200 + ): result = json.loads(response.read()) break retry += 1 diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py index 4c3c771b..7cc2c3bc 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py @@ -132,6 +132,7 @@ def get_current_metadata_for_address( WHERE address = :address AND label = :label + AND label_data ->>'metadata' != 'null' ORDER BY label_data ->> 'token_id' ASC, block_number :: INT DESC;