crawl less often, and fewer statuses

.gitignore

@@ -1,7 +1,6 @@
 *.csv
 .idea/
 *.gexf
-data/
 *.class
 
 backend/.sobelow

backend/config/config.exs

@@ -59,9 +59,9 @@ config :backend, Mastodon.Messenger,
 
 config :backend, :crawler,
   status_age_limit_days: 28,
-  status_count_limit: 5000,
+  status_count_limit: 1000,
   personal_instance_threshold: 10,
-  crawl_interval_mins: 30,
+  crawl_interval_mins: 60,
   crawl_workers: 100,
   blacklist: [
     # spam
@@ -100,6 +100,8 @@ config :backend, Backend.Scheduler,
     {"0 */3 * * *", {Backend.Scheduler, :check_for_spam_instances, []}}
   ]
 
+config :backend, :environment, Mix.env()
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

backend/lib/backend/application.ex

@@ -31,7 +31,8 @@ defmodule Backend.Application do
     ]
 
     children =
-      if Enum.member?(["true", 1, "1"], System.get_env("SKIP_CRAWL")) or Mix.env() == :test do
+      if Enum.member?(["true", 1, "1"], System.get_env("SKIP_CRAWL")) or
+           Application.get_env(:backend, :environment) == :test do
         children
       else
         children ++ [Backend.Crawler.StaleInstanceManager]