Delayed Job: Add ability to specify queues per worker

cookbooks/delayed_job4/recipes/default.rb

@@ -17,27 +17,31 @@
     mode 0755
   end
 
-  node['delayed_job4']['applications'].each do |app_name|
-    node['delayed_job4']['worker_count'].times do |count|
-      template "/etc/monit.d/delayed_job#{count+1}.#{app_name}.monitrc" do
-        source "dj.monitrc.erb"
-        owner "root"
-        group "root"
-        mode 0644
-        variables({
-          :app_name => app_name,
-          :user => node[:owner_name],
-          :worker_name => "#{app_name}_delayed_job#{count+1}",
-          :framework_env => node[:dna][:environment][:framework_env],
-          :worker_memory => node['delayed_job4']['worker_memory']
-        })
-      end
-    end
+  # Only one app per env by now
+  # Improvement:  have it loop across all apps on the env
+
+  app_name = node['delayed_job4']['applications'].first
 
-    execute "monit reload" do
-       action :run
-       epic_fail true
-    end
+  # The queues per worker definition is send as-is to the .erb template
+  # to be processed there
+
+  template "/etc/monit.d/delayed_job.#{app_name}.monitrc" do
+    source "dj.monitrc.erb"
+    owner "root"
+    group "root"
+    mode 0644
+    variables({
+                :app_name => app_name,
+                :user => node[:owner_name],
+                :queues => node['delayed_job4']['queues'],
+                :framework_env => node[:dna][:environment][:framework_env],
+                :worker_memory => node['delayed_job4']['worker_memory']
+              })
+  end
 
+  execute "monit reload" do
+    action :run
+    epic_fail true
   end
+
 end

cookbooks/delayed_job4/templates/default/dj.monitrc.erb

@@ -1,6 +1,14 @@
-check process <%= @worker_name %>
-  with pidfile /var/run/engineyard/dj/<%= @app_name %>/dj_<%= @worker_name %>.pid
-  start program = "/engineyard/custom/dj <%= @app_name %> start <%= @framework_env %> <%= @worker_name %>" with timeout 90 seconds
-  stop program = "/engineyard/custom/dj <%= @app_name %> stop <%= @framework_env %> <%= @worker_name %>" with timeout 90 seconds
-  if totalmem is greater than <%= @worker_memory %> MB then restart # eating up memory?
-  group dj_<%= @app_name %>
+<% @queues.each do |queue_name, workers| %>
+
+   <% for count in 1..workers do %>
+    check process <%= queue_name %>_<%= count %>
+    with pidfile /var/run/engineyard/dj/<%= @app_name %>/dj_<%= queue_name %>_<%= count %>.pid
+    start program = "/usr/bin/env QUEUES=<%= queue_name %> /engineyard/custom/dj <%= @app_name %> start <%= @framework_env %> <%= queue_name %>_<%= count %>" with timeout 90 seconds
+    stop program = "/engineyard/custom/dj <%= @app_name %> stop <%= @framework_env %> <%= queue_name %>" with timeout 90 seconds
+    if totalmem is greater than <%= @worker_memory %> MB then restart # eating up memory?
+    group dj_<%= @app_name %>
+  <% end %>
+
+<% end %>
+
+

custom-cookbooks/delayed_job4/cookbooks/custom-delayed_job4/attributes/default.rb

@@ -1,4 +1,24 @@
-# default['delayed_job4']['is_dj_instance'] = (node['dna']['instance_role'] == 'util' && node['dna']['name'] == 'delayed_job')
+default['delayed_job4']['is_dj_instance'] = (node['dna']['instance_role'] == 'util' && node['dna']['name'] == 'delayed_job')
 # default['delayed_job4']['applications'] = %w[todo]
-# default['delayed_job4']['worker_count'] = 4
-# default['delayed_job4']['worker_memory'] = 300
+#default['delayed_job4']['worker_count'] = 4
+
+# The config below is for a single app per environment
+# Improvement:  make it so that DJ for multiple apps per env can be configured
+# in a way that each has it own set of workers with its own memory limit, and then queues per worker
+
+# Memory per worker, going above this value will trigger monit to restart the worker
+# Same value for all workers
+# Improvement:  set the memory limit per worker
+default['delayed_job4']['worker_memory'] = 2000
+
+# Defining number of workers per queue
+# so to spin up a process for each worker and configure queues to be run for ti
+# values below are examples
+
+default['delayed_job4']['queues'] = {
+  # :queue_name => number of workers
+  :my_queue => 3,
+  :his_queue => 1,
+  :her_queue => 2,
+  :their_queue => 1
+}