Crawlers server for health checks

crawlers/deploy/deploy.bash

@@ -19,6 +19,7 @@ ETHEREUM_SYNCHRONIZE_SERVICE="ethereum-synchronize.service"
 ETHEREUM_TRENDING_SERVICE="ethereum-trending.service"
 ETHEREUM_TRENDING_TIMER="ethereum-trending.service"
 ETHEREUM_TXPOOL_SERVICE="ethereum-txpool.service"
+SERVICE_FILE="moonstreamcrawlers.service"
 
 set -eu
 
@@ -30,6 +31,14 @@ cd "${APP_CRAWLERS_DIR}/ethtxpool"
 HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/ethtxpool/ethtxpool" "${APP_CRAWLERS_DIR}/ethtxpool/main.go"
 cd "${EXEC_DIR}"
 
+echo
+echo
+echo "Building executable server of moonstreamcrawlers with Go"
+EXEC_DIR=$(pwd)
+cd "${APP_CRAWLERS_DIR}/server"
+HOME=/root /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/server/moonstreamcrawlers" "${APP_CRAWLERS_DIR}/server/main.go"
+cd "${EXEC_DIR}"
+
 echo
 echo
 echo "Updating Python dependencies"
@@ -82,3 +91,12 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}"
 cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE}" "/etc/systemd/system/${ETHEREUM_TXPOOL_SERVICE}"
 systemctl daemon-reload
 systemctl restart "${ETHEREUM_TXPOOL_SERVICE}"
+
+echo
+echo
+echo "Replacing existing moonstreamcrawlers service definition with ${SERVICE_FILE}"
+chmod 644 "${SCRIPT_DIR}/${SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${SERVICE_FILE}" "/etc/systemd/system/${SERVICE_FILE}"
+systemctl daemon-reload
+systemctl restart "${SERVICE_FILE}"
+systemctl status "${SERVICE_FILE}"

crawlers/deploy/moonstreamcrawlers.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=moonstreamcrawlers-service
+After=network.target
+
+[Service]
+User=ubuntu
+Group=www-data
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/server
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream/crawlers/server/moonstreamcrawlers -host 0.0.0.0 -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"
+SyslogIdentifier=moonstreamcrawlers
+
+[Install]
+WantedBy=multi-user.target

crawlers/server/dev.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+# Expects access to Python environment with the requirements for this project installed.
+set -e
+
+MOONSTREAM_CRAWLERS_SERVER_HOST="${MOONSTREAM_CRAWLERS_SERVER_HOST:-0.0.0.0}"
+MOONSTREAM_CRAWLERS_SERVER_PORT="${MOONSTREAM_CRAWLERS_SERVER_PORT:-8080}"
+
+go run main.go -host "${MOONSTREAM_CRAWLERS_SERVER_HOST}" -port "${MOONSTREAM_CRAWLERS_SERVER_PORT}"

crawlers/server/go.mod

@@ -0,0 +1,3 @@
+module moonstreamdb
+
+go 1.17

crawlers/server/main.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+)
+
+var MOONSTREAM_IPC_PATH = os.Getenv("MOONSTREAM_DB_URI")
+
+type GethEthSyncingResponse struct {
+	CurrentBlock string `json:"currentBlock"`
+}
+
+type GethResponse struct {
+	Result GethEthSyncingResponse `json:"result"`
+}
+
+type PingGethResponse struct {
+	Status       string `json:"status"`
+	CurrentBlock string `json:"current_block"`
+}
+
+type PingResponse struct {
+	Status string `json:"status"`
+}
+
+func ping(w http.ResponseWriter, req *http.Request) {
+	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+
+	w.Header().Set("Content-Type", "application/json")
+	response := PingResponse{Status: "ok"}
+	json.NewEncoder(w).Encode(response)
+}
+
+func pingGeth(w http.ResponseWriter, req *http.Request) {
+	log.Printf("%s, %s, %q", req.RemoteAddr, req.Method, req.URL.String())
+
+	postBody, err := json.Marshal(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"method":  "eth_syncing",
+		"id":      1,
+	})
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	gethResponse, err := http.Post(MOONSTREAM_IPC_PATH, "application/json",
+		bytes.NewBuffer(postBody))
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	defer gethResponse.Body.Close()
+
+	gethResponseBody, err := ioutil.ReadAll(gethResponse.Body)
+	if err != nil {
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	var obj GethResponse
+	_ = json.Unmarshal(gethResponseBody, &obj)
+
+	w.Header().Set("Content-Type", "application/json")
+	response := PingGethResponse{Status: "ok", CurrentBlock: obj.Result.CurrentBlock}
+	json.NewEncoder(w).Encode(response)
+}
+
+func main() {
+	var listenAddr string
+	var listenPort string
+	flag.StringVar(&listenAddr, "host", "127.0.0.1", "Server listen address")
+	flag.StringVar(&listenPort, "port", "8080", "Server listen port")
+	flag.Parse()
+
+	address := listenAddr + ":" + listenPort
+	log.Printf("Starting server at %s\n", address)
+
+	http.HandleFunc("/ping", ping)
+	http.HandleFunc("/ping/geth", pingGeth)
+
+	http.ListenAndServe(address, nil)
+}

crawlers/server/sample.env

@@ -0,0 +1,2 @@
+export MOONSTREAM_CRAWLERS_SERVER_PORT="8080"
+export MOONSTREAM_IPC_PATH=null

db/server/main.go

@@ -13,11 +13,6 @@ import (
 
 var MOONSTREAM_DB_URI = os.Getenv("MOONSTREAM_DB_URI")
 
-type Error interface {
-	error
-	Status() int
-}
-
 type PingResponse struct {
 	Status string `json:"status"`
 }