From c33d033ad9e0514116a207ad0932a5e91bd25827 Mon Sep 17 00:00:00 2001
From: kompotkot <zenitsws@gmail.com>
Date: Sun, 6 Mar 2022 18:51:36 +0000
Subject: [PATCH] Smart verification humbug report counter

---
 crawlers/ldb/cmd/actions.go      | 16 +++++++---------
 crawlers/ldb/cmd/cli.go          | 19 +++++++++++++++++++
 crawlers/ldb/cmd/reporter.go     |  7 ++++---
 crawlers/ldb/configs/settings.go | 13 -------------
 crawlers/ldb/sample.env          |  1 -
 5 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/crawlers/ldb/cmd/actions.go b/crawlers/ldb/cmd/actions.go
index 9345c43d..5c89ebc0 100644
--- a/crawlers/ldb/cmd/actions.go
+++ b/crawlers/ldb/cmd/actions.go
@@ -47,6 +47,7 @@ func add(blockchain string, blockNumbers []uint64) error {
 
 		fmt.Printf("Processed block number: %d\r", bn)
 	}
+
 	return nil
 }
 
@@ -59,16 +60,18 @@ func show(blockNumbers []uint64) error {
 			continue
 		}
 
-		chainTxs := localConnections.getChainTxs(block.Hash(), bn)
+		// chainTxs := localConnections.getChainTxs(block.Hash(), bn)
 
 		var txs []common.Hash
-		for _, tx := range chainTxs {
-			txs = append(txs, tx.Hash())
-		}
+		// for _, tx := range chainTxs {
+		// 	txs = append(txs, tx.Hash())
+		// }
 
 		fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs)
 	}
 
+	fmt.Println("new range")
+
 	return nil
 }
 
@@ -122,10 +125,5 @@ func verify(blockchain string, blockNumbers []uint64) error {
 		cnt++
 	}
 
-	err := humbugReporter.submitReport(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
-	if err != nil {
-		return fmt.Errorf("Unable to send humbug report: %v", err)
-	}
-
 	return nil
 }
diff --git a/crawlers/ldb/cmd/cli.go b/crawlers/ldb/cmd/cli.go
index fdcb9388..89707fea 100644
--- a/crawlers/ldb/cmd/cli.go
+++ b/crawlers/ldb/cmd/cli.go
@@ -8,12 +8,14 @@ import (
 	"sort"
 	"strconv"
 
+	"github.com/bugout-dev/moonstream/crawlers/ldb/configs"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/google/uuid"
 	"gopkg.in/urfave/cli.v1"
 )
 
 var (
+	// Block steps used to prevent long executor tasks and data loss possibility
 	BlockNumberStep = uint64(1000)
 
 	BlockchainFlag = cli.StringFlag{
@@ -188,11 +190,28 @@ func processVerifyCommand(ctx *cli.Context) error {
 		return fmt.Errorf("Unable to set database connection: %v", err)
 	}
 
+	cnt := uint64(0)
+	reportStart := uint64(start)
 	for blocks := range BlockYield(start, end, BlockNumberStep) {
 		err = verify(blockchain, blocks)
 		if err != nil {
 			return fmt.Errorf("Error occurred due verify acction: %v", err)
 		}
+
+		cnt += BlockNumberStep
+		if cnt >= configs.BLOCK_RANGE_REPORT {
+			err := humbugReporter.submitReport(reportStart, blocks[len(blocks)-1]+1, "")
+			if err != nil {
+				return fmt.Errorf("Unable to send humbug report: %v", err)
+			}
+			reportStart = blocks[len(blocks)-1] + 1
+			cnt = 0
+		}
+	}
+
+	err = humbugReporter.submitReport(start, end, "Total ")
+	if err != nil {
+		return fmt.Errorf("Unable to send humbug report: %v", err)
 	}
 
 	localConnections.Chain.Stop()
diff --git a/crawlers/ldb/cmd/reporter.go b/crawlers/ldb/cmd/reporter.go
index 25e3ab5b..bd4e2cd2 100644
--- a/crawlers/ldb/cmd/reporter.go
+++ b/crawlers/ldb/cmd/reporter.go
@@ -24,14 +24,15 @@ func setHumbugClient(sessionID string) error {
 	return nil
 }
 
-func (r *HumbugReporter) submitReport(start, end uint64) error {
+// Publish report with verified blocks to entry
+func (r *HumbugReporter) submitReport(start, end uint64, prefix string) error {
 	content, err := json.Marshal(corruptBlocks)
 	if err != nil {
 		return fmt.Errorf("Unable to marshal to json: %v", err)
 	}
 
 	report := humbug.Report{
-		Title:   fmt.Sprintf("LDB verifier %d-%d", start, end),
+		Title:   fmt.Sprintf("%sLDB verifier %d-%d", prefix, start, end),
 		Content: string(content),
 		Tags: []string{
 			fmt.Sprintf("start:%d", start),
@@ -39,7 +40,7 @@ func (r *HumbugReporter) submitReport(start, end uint64) error {
 		},
 	}
 	r.Reporter.Publish(report)
-	fmt.Printf("Error for range %d-%d published\n", start, end)
+	fmt.Printf("%sLDB report for range %d-%d published\n", prefix, start, end)
 
 	return nil
 }
diff --git a/crawlers/ldb/configs/settings.go b/crawlers/ldb/configs/settings.go
index c1e6fd8d..19914539 100644
--- a/crawlers/ldb/configs/settings.go
+++ b/crawlers/ldb/configs/settings.go
@@ -1,9 +1,7 @@
 package configs
 
 import (
-	"log"
 	"os"
-	"strconv"
 	"time"
 )
 
@@ -17,14 +15,3 @@ var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
 // Humber configs
 var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID")
 var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN")
-var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT")
-
-func init() {
-	if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" {
-		ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 16, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		BLOCK_RANGE_REPORT = ldbRangeReport
-	}
-}
diff --git a/crawlers/ldb/sample.env b/crawlers/ldb/sample.env
index 807ca4ff..449b363a 100644
--- a/crawlers/ldb/sample.env
+++ b/crawlers/ldb/sample.env
@@ -1,4 +1,3 @@
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
 export HUMBUG_LDB_CLIENT_ID="<client_id_of_crawl_service>"
 export HUMBUG_LDB_TOKEN="<generate_humbug_integration_with_token_at_https://bugout.dev/account/teams>"
-export HUMBUG_LDB_BLOCK_RANGE_REPORT=100000