fix crawls of well-connected instances

.github/workflows/deploy.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Build & push
         uses: depot/build-push-action@v1
         with:
-          project: 3tz1wxj8cr
+          project: rktsv8c4sk
           context: backend
           push: true
           tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -5,6 +5,7 @@ data/
 *.class
 
 backend/.sobelow
+node_modules/
 
 # Environments
 .env

backend/lib/backend/crawler/crawler.ex

@@ -235,9 +235,12 @@ defmodule Backend.Crawler do
         Enum.map(result.federation_restrictions, fn {domain, _restriction_type} -> domain end)
       )
       |> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now, next_crawl: now})
+      |> Enum.chunk_every(5000)
 
-    Instance
+    new_instances
-    |> Repo.insert_all(new_instances, on_conflict: :nothing, conflict_target: :domain)
+    |> Enum.each(fn chunk ->
+      Repo.insert_all(Instance, chunk, on_conflict: :nothing, conflict_target: :domain)
+    end)
 
     Repo.transaction(fn ->
       ## Save peer relationships ##
@@ -274,9 +277,10 @@ defmodule Backend.Crawler do
             updated_at: now
           }
         )
+        |> Enum.chunk_every(5000)
 
-      InstancePeer
+      new_instance_peers
-      |> Repo.insert_all(new_instance_peers)
+      |> Enum.each(fn chunk -> Repo.insert_all(InstancePeer, chunk) end)
     end)
 
     ## Save federation restrictions ##