Smart verification humbug report counter

crawlers/ldb/cmd/actions.go

@@ -47,6 +47,7 @@ func add(blockchain string, blockNumbers []uint64) error {
 
 		fmt.Printf("Processed block number: %d\r", bn)
 	}
+
 	return nil
 }
 
@@ -59,16 +60,18 @@ func show(blockNumbers []uint64) error {
 			continue
 		}
 
-		chainTxs := localConnections.getChainTxs(block.Hash(), bn)
+		// chainTxs := localConnections.getChainTxs(block.Hash(), bn)
 
 		var txs []common.Hash
-		for _, tx := range chainTxs {
-			txs = append(txs, tx.Hash())
-		}
+		// for _, tx := range chainTxs {
+		// 	txs = append(txs, tx.Hash())
+		// }
 
 		fmt.Printf("Block %d block with hash: %s and transactions: %s\n", block.Number(), block.Hash().String(), txs)
 	}
 
+	fmt.Println("new range")
+
 	return nil
 }
 
@@ -122,10 +125,5 @@ func verify(blockchain string, blockNumbers []uint64) error {
 		cnt++
 	}
 
-	err := humbugReporter.submitReport(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
-	if err != nil {
-		return fmt.Errorf("Unable to send humbug report: %v", err)
-	}
-
 	return nil
 }

crawlers/ldb/cmd/cli.go

@@ -8,12 +8,14 @@ import (
 	"sort"
 	"strconv"
 
+	"github.com/bugout-dev/moonstream/crawlers/ldb/configs"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/google/uuid"
 	"gopkg.in/urfave/cli.v1"
 )
 
 var (
+	// Block steps used to prevent long executor tasks and data loss possibility
 	BlockNumberStep = uint64(1000)
 
 	BlockchainFlag = cli.StringFlag{
@@ -188,11 +190,28 @@ func processVerifyCommand(ctx *cli.Context) error {
 		return fmt.Errorf("Unable to set database connection: %v", err)
 	}
 
+	cnt := uint64(0)
+	reportStart := uint64(start)
 	for blocks := range BlockYield(start, end, BlockNumberStep) {
 		err = verify(blockchain, blocks)
 		if err != nil {
 			return fmt.Errorf("Error occurred due verify acction: %v", err)
 		}
+
+		cnt += BlockNumberStep
+		if cnt >= configs.BLOCK_RANGE_REPORT {
+			err := humbugReporter.submitReport(reportStart, blocks[len(blocks)-1]+1, "")
+			if err != nil {
+				return fmt.Errorf("Unable to send humbug report: %v", err)
+			}
+			reportStart = blocks[len(blocks)-1] + 1
+			cnt = 0
+		}
+	}
+
+	err = humbugReporter.submitReport(start, end, "Total ")
+	if err != nil {
+		return fmt.Errorf("Unable to send humbug report: %v", err)
 	}
 
 	localConnections.Chain.Stop()

crawlers/ldb/cmd/reporter.go

@@ -24,14 +24,15 @@ func setHumbugClient(sessionID string) error {
 	return nil
 }
 
-func (r *HumbugReporter) submitReport(start, end uint64) error {
+// Publish report with verified blocks to entry
+func (r *HumbugReporter) submitReport(start, end uint64, prefix string) error {
 	content, err := json.Marshal(corruptBlocks)
 	if err != nil {
 		return fmt.Errorf("Unable to marshal to json: %v", err)
 	}
 
 	report := humbug.Report{
-		Title:   fmt.Sprintf("LDB verifier %d-%d", start, end),
+		Title:   fmt.Sprintf("%sLDB verifier %d-%d", prefix, start, end),
 		Content: string(content),
 		Tags: []string{
 			fmt.Sprintf("start:%d", start),
@@ -39,7 +40,7 @@ func (r *HumbugReporter) submitReport(start, end uint64) error {
 		},
 	}
 	r.Reporter.Publish(report)
-	fmt.Printf("Error for range %d-%d published\n", start, end)
+	fmt.Printf("%sLDB report for range %d-%d published\n", prefix, start, end)
 
 	return nil
 }

crawlers/ldb/configs/settings.go

@@ -1,9 +1,7 @@
 package configs
 
 import (
-	"log"
 	"os"
-	"strconv"
 	"time"
 )
 
@@ -17,14 +15,3 @@ var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
 // Humber configs
 var HUMBUG_LDB_CLIENT_ID = os.Getenv("HUMBUG_LDB_CLIENT_ID")
 var HUMBUG_LDB_TOKEN = os.Getenv("HUMBUG_LDB_TOKEN")
-var HUMBUG_LDB_BLOCK_RANGE_REPORT = os.Getenv("HUMBUG_LDB_BLOCK_RANGE_REPORT")
-
-func init() {
-	if HUMBUG_LDB_BLOCK_RANGE_REPORT != "" {
-		ldbRangeReport, err := strconv.ParseUint(HUMBUG_LDB_BLOCK_RANGE_REPORT, 16, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		BLOCK_RANGE_REPORT = ldbRangeReport
-	}
-}

crawlers/ldb/sample.env

@@ -1,4 +1,3 @@
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
 export HUMBUG_LDB_CLIENT_ID="<client_id_of_crawl_service>"
 export HUMBUG_LDB_TOKEN="<generate_humbug_integration_with_token_at_https://bugout.dev/account/teams>"
-export HUMBUG_LDB_BLOCK_RANGE_REPORT=100000