Experimental import limits

Related to #17

lib/toniq/job_importer.ex

@@ -18,10 +18,22 @@ defmodule Toniq.JobImporter do
   defp import_jobs(enabled: false), do: nil
   defp import_jobs(enabled: true) do
     incoming_jobs
+    |> Enum.take(jobs_to_import_count)
     |> log_import
     |> Enum.each(&import_job/1)
   end
 
+  defp jobs_to_import_count do
+    max_count = 500 # this would be in config
+    diff = max_count - Toniq.JobPersistence.jobs_count
+
+    if diff < 0 do
+      0
+    else
+      diff
+    end
+  end
+
   defp incoming_jobs do
     Toniq.JobPersistence.incoming_jobs
   end

lib/toniq/job_persistence.ex

@@ -23,6 +23,8 @@ defmodule Toniq.JobPersistence do
   """
   def jobs(identifier \\ default_identifier), do: load_jobs(jobs_key(identifier), identifier)
 
+  def jobs_count(identifier \\ default_identifier), do: scard(redis, jobs_key(identifier)) |> String.to_integer
+
   @doc """
   Returns all incoming jobs (used for failover).
   """