Added function/event crawler for 4byte.directory

crawlers/4byteCrawler/4byteDirectoryCrawler.js

@@ -0,0 +1,42 @@
+const fetch = require('node-fetch')
+const fs = require("fs")
+
+   
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function makeRequest(url, sleepTime) {
+    if (sleepTime)
+        sleep(sleepTime)
+    let response = await fetch(url);
+    let json = await response.json();
+    return json;
+}
+
+async function crawlFunctionSignatures() {
+    let url = "https://www.4byte.directory/api/v1/signatures/";
+    let response = await makeRequest(url)
+    let signatures = response.results;
+    while (response.next) {
+        response = await makeRequest(response.next)
+        signatures = signatures.concat(response.results)
+        console.log(`already crawled : ${signatures.length}`)
+    }
+    fs.writeFileSync("function_signatures.json", JSON.stringify(signatures))
+}
+
+async function crawlEventSignatures() {
+    let url = "https://www.4byte.directory/api/v1/event-signatures/";
+    let response = await makeRequest(url)
+    let signatures = response.results;
+    while (response.next) {
+        response = await makeRequest(response.next)
+        signatures = signatures.concat(response.results)
+        console.log(`already crawled : ${signatures.length}`)
+    }
+    fs.writeFileSync("event_signatures.json", JSON.stringify(signatures))
+}
+
+//crawlFunctionSignatures()
+//crawlEventSignatures();

crawlers/4byteCrawler/json2Sqlite.js

@@ -0,0 +1,57 @@
+const sqlite3 = require('sqlite3').verbose()
+const fs = require('fs')
+let db = new sqlite3.Database('./signatures.db', sqlite3.OPEN_READWRITE, (err) => {
+    if (err) {
+      console.error(err.message);
+    }
+    
+});
+
+function put_func_signatures_to_db() {
+    db.serialize(() => {
+        db.run('create table if not exists '
+            + 'function_signatures('
+            + 'id numeric primary key,'
+            + 'text_signature text,'
+            + 'hex_signature text)')
+        
+        let stmt = db.prepare('insert into function_signatures values (?, ?, ?)')
+        let function_signatures = JSON.parse(fs.readFileSync("./function_signatures.json"))
+        function_signatures.forEach((item) => {
+            try {
+                stmt.run([item.id, item.text_signature, item.hex_signature])
+                
+            }
+            catch(err) {
+                console.log(item)
+                console.log(err)
+            }
+        })
+    })
+}
+
+function put_event_signatures_to_db() {
+    db.serialize(() => {
+        db.run('create table if not exists '
+            + 'event_signatures('
+            + 'id numeric primary key,'
+            + 'text_signature text,'
+            + 'hex_signature text)')
+        
+        let stmt = db.prepare('insert into event_signatures values (?, ?, ?)')
+        let function_signatures = JSON.parse(fs.readFileSync("./event_signatures.json"))
+        function_signatures.forEach((item) => {
+            try {
+                stmt.run([item.id, item.text_signature, item.hex_signature])
+                
+            }
+            catch(err) {
+                console.log(item)
+                console.log(err)
+            }
+        })
+    })
+}
+
+//put_func_signatures_to_db()
+//put_event_signatures_to_db()

crawlers/4byteCrawler/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "signature_crawler",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "csv-parser": "^3.0.0",
+    "node-fetch": "^2.6.1",
+    "sqlite3": "^5.0.2"
+  }
+}