Log action queue size on channel disconnect

Ticket: https://fatsoma.atlassian.net/browse/ENG-549

Print and report the main thread action queue size when a channel is closed on error. This is to check the main thread is effectively servicing the queued messages sent from the workers.

Author: brentfatsoma

lib/hutch/channel_broker.rb

@@ -66,9 +66,18 @@ def open_channel
 
         # on_error handler logs and notifies any unhandled channel errors
         ch.on_error do |channel, method|
-          logger.error "Channel error [channel=#{channel.inspect}, method=#{method.inspect}, active=#{active}]"
-
-          context = {method: method.inspect}
+          action_queue_size = Thread.main[:action_queue].size
+
+          logger.error("Channel error [channel=#{channel.inspect}, " \
+           "method=#{method.inspect}, " \
+           "active=#{active}, " \
+           "action_queue_size=#{action_queue_size}]"
+          )
+
+          context = {
+            method: method.inspect,
+            action_queue_size: action_queue_size
+          }
           if method.is_a?(AMQ::Protocol::Channel::Close)
             error_message = method.reply_text
             context[:reply_code] = method.reply_code

spec/hutch/channel_broker_spec.rb

@@ -5,6 +5,9 @@
   before do
     Hutch::Config.initialize(client_logger: Hutch::Logging.logger)
     @config = Hutch::Config.to_hash
+    stub_const('Honeybadger', honey_badger)
+    allow(honey_badger).to receive(:add_breadcrumb)
+    allow(honey_badger).to receive(:notify)
   end
   let!(:config) { @config }
   after do
@@ -13,6 +16,7 @@
   end
   let(:connection) { double(:connection) }
   let(:channel) { double(:channel) }
+  let(:honey_badger) { double(:honey_badger) }
 
   subject(:channel_broker) { Hutch::ChannelBroker.new(connection, config) }
 
@@ -143,17 +147,16 @@ def test_on_error(*args)
         def confirm_select; end
       end.new
     end
-    let(:honey_badger) { double(:honey_badger) }
     let(:config) { {} }
     let(:method) { Class.new }
+    let(:action_queue_size) { 5 }
 
     before do
-      stub_const('Honeybadger', honey_badger)
-      allow(honey_badger).to receive(:add_breadcrumb)
-      allow(honey_badger).to receive(:notify)
       allow(connection).to receive(:create_channel).and_return(channel)
       allow(connection).to receive(:prefetch_channel)
       allow(channel).to receive(:confirm_select)
+      allow(Thread.main).to receive(:[]).with(:action_queue)
+        .and_return(double(size: action_queue_size))
     end
 
     subject { channel_broker.open_channel }
@@ -204,7 +207,8 @@ def confirm_select; end
         let(:context) do
           {
             reply_code: reply_code,
-            method: method.inspect
+            method: method.inspect,
+            action_queue_size: action_queue_size
           }
         end
 
@@ -225,7 +229,8 @@ def confirm_select; end
         end
         let(:context) do
           {
-            method: method.inspect
+            method: method.inspect,
+            action_queue_size: action_queue_size
           }
         end