Add deletion step.

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -19,6 +19,7 @@ from .db import (
     get_current_metadata_for_address,
     get_tokens_wich_maybe_updated,
     metadata_to_label,
+    clean_labels_from_db,
 )
 from ..settings import (
     MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
@@ -141,23 +142,32 @@ def parse_metadata(
                 for i in range(0, len(tokens_uri_by_address[address]), batch_size)
             ]:
                 writed_labels = 0
-                for token_uri_data in requests_chunk:
+                db_session.commit()
 
-                    if token_uri_data.token_id not in parsed_with_leak:
-                        metadata = crawl_uri(token_uri_data.token_uri)
+                with db_session.begin():
+                    for token_uri_data in requests_chunk:
 
-                        db_session.add(
-                            metadata_to_label(
-                                blockchain_type=blockchain_type,
-                                metadata=metadata,
-                                token_uri_data=token_uri_data,
+                        if token_uri_data.token_id not in parsed_with_leak:
+                            metadata = crawl_uri(token_uri_data.token_uri)
+
+                            db_session.add(
+                                metadata_to_label(
+                                    blockchain_type=blockchain_type,
+                                    metadata=metadata,
+                                    token_uri_data=token_uri_data,
+                                )
                             )
-                        )
-                        writed_labels += 1
+                            writed_labels += 1
 
-                if writed_labels > 0:
-                    commit_session(db_session)
-                    logger.info(f"Write {writed_labels} labels for {address}")
+                    if writed_labels > 0:
+                        clean_labels_from_db(
+                            db_session=db_session,
+                            blockchain_type=blockchain_type,
+                            address=address,
+                        )
+                        logger.info(f"Write {writed_labels} labels for {address}")
+
+                # trasaction is commited here
 
     finally:
         db_session.close()

crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py

@@ -210,3 +210,43 @@ def get_tokens_wich_maybe_updated(
     result = [data[0] for data in tokens]
 
     return result
+
+
+def clean_labels_from_db(
+    db_session: Session, blockchain_type: AvailableBlockchainType, address: str
+):
+    """
+    Remove existing labels.
+    """
+
+    label_model = get_label_model(blockchain_type)
+
+    table = label_model.__tablename__
+
+    db_session.execute(
+        """ 
+        WITH lates_token_metadata AS (
+            SELECT
+                DISTINCT ON (label_data->>'token_id') label_data->>'token_id' AS token_id,
+                id as id,
+                block_number as block_number
+            FROM
+                {}
+            WHERE
+                label=:label
+                AND address=:address
+            ORDER BY
+                label_data->>'token_id' ASC,
+                block_number DESC
+        )
+        DELETE FROM
+            {} USING lates_token_metadata
+        WHERE
+            label=:label
+            AND address=:address
+            AND polygon_labels.id  not in (select id from lates_token_metadata) RETURNING polygon_labels.block_number;
+    """.format(
+            table, table
+        ),
+        {"address": address, "label": METADATA_CRAWLER_LABEL},
+    )