New BAC method for failed items

New BAC methods return the list of failed
or successful items directly. The old
method 'indices_of_failures' was removed,
and the retry mechanism now calls 'failed_items'.

BrainPortal/app/models/background_activity.rb

@@ -49,7 +49,9 @@
 #   current_item        # increasing counter within items, starts at 0
 #   num_successes       # as processing goes on records successes
 #   num_failures        # same for failures
-#   messages            # array of string messages, aligned with items (serialized)
+#   messages            # array of string messages, aligned with items (serialized);
+#                       # by convention, the message nil is used to indicate an item
+#                       # was processed properly
 #   options             # hash of arbitrary options, specific to the BAC class
 #   created_at
 #   updated_at
@@ -258,10 +260,14 @@ def process_next_item
     ok,message = nil,nil
     begin
       ok,message = self.process(item)
-    rescue => ex
+    rescue ActiveRecord::RecordNotFound
       ok      = false
+      message = 'NotFound'
+    rescue => ex
+      ok = false
       better_message = ex.message
-      better_message.sub!(/\[WHERE.*/, "") if ex.is_a?(ActiveRecord::RecordNotFound)
+      # place code here to simplify other common exception messages
+      better_message.sub!(/Internal error: Cannot create or find SyncStatus object for userfile.*/, "No SyncStatus")
       message = "#{ex.class}: #{better_message}"
     end
 
@@ -321,13 +327,41 @@ def uniq_counted_messages
     counts[0..4].map { |m,c| "#{c}x #{m.strip}" }
   end
 
+  # Returns an array of all items that have been
+  # processed properly; by default this is the
+  # equivalent of selected_items_from_nil_messages
+  # but subclasses can override this behavior.
+  def successful_items
+    selected_items_from_nil_messages()
+  end
+
+  # Returns an array of all items that have failed
+  # to process properly; by default this is the
+  # equivalent of selected_items_from_present_messages
+  # but subclasses can override this behavior.
+  def failed_items
+    selected_items_from_present_messages()
+  end
+
+  # Returns the tail end of the items list
+  # containing all the items yet to be processed,
+  # that is, items with index >= current_item
+  def remaining_items
+    self.items[self.current_item..-1] || []
+  end
+
   # Utility method that returns an ordered subset of
   # the items list, but only for items where
   # the associated message (in the +messages+ list)
   # match the regular expression +regex+ . If
   # +regex+ is nil, will return the subset of items where
   # the associated messages is nil.
   #
+  # Note that the official interface for extracting
+  # successful and failed items are in the methods
+  # +failed_items+ and +successful_items+.  These two methods
+  # actually indirectly invoke this method here.
+  #
   # By convention, a succesfully processed item is
   # generally recorded as a nil in the message array,
   # and a failed or skipped item is recorded with
@@ -367,8 +401,10 @@ def selected_items_from_messages_matching(regex)
   #
   #   selected_items_from_messages_matching(nil)
   #
-  # thus returning the subset of items that were
-  # successfully processed.
+  # thus returning the subset of items that recorded
+  # a message value +nil+ (usually, success).
+  #
+  # See also the method +successful_items+ .
   def selected_items_from_nil_messages
     selected_items_from_messages_matching(nil)
   end
@@ -378,7 +414,9 @@ def selected_items_from_nil_messages
   #   selected_items_from_messages_matching(/./)
   #
   # thus returning the subset of items that
-  # failed for any reason.
+  # retured a string message (usually, a failure).
+  #
+  # See also the method +failed_items+ .
   def selected_items_from_present_messages
     selected_items_from_messages_matching(/./)
   end
@@ -623,8 +661,7 @@ def zap_retry_attributes
   # 5- internally double retry_delay (so that at the next retry, the
   # rescheduling happens even later)
   #
-  # For step 3 to work, a BAC's class must implement the method indices_of_failures();
-  # if this method doesn't exist, retrying will fail.
+  # For step 3 to work, a BAC's class must implement the method failed_items() properly.
   #
   # If everything is ok, this method returns the backup object created
   # in step 1, otherwise it returns nil.
@@ -633,8 +670,7 @@ def setup_retry!
     # First, a few consistency checks and cleanup at the same time.
     if (self.status !~ /Failed|PartiallyCompleted/) ||
        (self.retry_count.blank?)                    ||
-       (self.retry_count < 1)                       ||
-       (self.status == "PartiallyCompleted" && ! self.respond_to?(:indices_of_failures))
+       (self.retry_count < 1)
       self.zap_retry_attributes
       self.save
       return nil
@@ -653,13 +689,7 @@ def setup_retry!
     # If the bac is in PartiallyCompleted state,
     # we extract just the elements to try again.
     if self.status = "PartiallyCompleted"
-      fail_idx = self.indices_of_failures rescue nil
-      if fail_idx.blank? || (!fail_idx.is_a?(Array)) # that's an error
-        self.internal_error!('Bad value for indices_of_failures()')
-        return nil
-      end
-      orig_items = self.items
-      new_items  = fail_idx.sort.map { |i| orig_items[i] }.compact
+      new_items = self.failed_items
       if new_items.blank?
         self.internal_error!('No items list after retry')
         return nil

BrainPortal/app/models/background_activity/check_missing_workdir.rb

@@ -58,6 +58,11 @@ def process(task_id)
     return [ true, "Adjusted" ]  # keyword used in after_last_item() below
   end
 
+  # Never consider anything as 'failed', to disable the retry mechanism
+  def failed_items
+    []
+  end
+
   def prepare_dynamic_items
     local_tasks_with_workdirs = CbrainTask
       .real_tasks

BrainPortal/app/models/background_activity/copy_file.rb

@@ -65,6 +65,18 @@ def process(item)
     return [ false, "Failed to copy '#{userfile.name}'" ]
   end
 
+  def successful_items
+    selected_items_from_messages_matching(/^(Skipped|\d+)$/) # Skipped or an ID
+  end
+
+  def failed_items
+    myitems = self.items
+    messages.each_with_index.map do |mess,idx|
+      next nil if mess.to_s.match(/^(Skipped|\d+)$/)
+      myitems[idx]
+    end.compact
+  end
+
   def prepare_dynamic_items
     populate_items_from_userfile_custom_filter
   end

BrainPortal/app/models/background_activity/copy_file_and_unregister.rb

@@ -33,11 +33,5 @@ def process(item)
     [ true, nil ]
   end
 
-  def indices_of_failures
-    self.messages.each_with_index.map do |message,idx|
-      idx if message.to_s.starts_with?('Failed')
-    end.compact
-  end
-
 end
 

BrainPortal/app/models/background_activity/random_activity.rb

@@ -69,10 +69,12 @@ def process(item)
     raise "Oh darn #{item} exception"
   end
 
-  def indices_of_failures
-    self.messages.each_with_index.map do |message,idx|
-      idx if ! message.to_s.starts_with?("Yeah")
-    end.compact
+  def failed_items
+    selected_items_from_messages_matching(/^Nope/)
+  end
+
+  def successful_items
+    selected_items_from_messages_matching(/^Yeah/)
   end
 
 end

BrainPortal/app/models/background_activity/register_and_copy_file.rb

@@ -56,5 +56,13 @@ def process(item)  # item is like "TextFile-abcd.xyz"
     return [ false, "Copy failed" ] # when provider_copy returns false
   end
 
+  def successful_items
+    selected_items_from_messages_matching(/^\d+$/) # must be a numberic ID
+  end
+
+  def failed_items
+    selected_items_from_messages_matching(/^\D/) # Anything starting with not a digit
+  end
+
 end
 

BrainPortal/app/models/background_activity/register_and_move_file.rb

@@ -55,5 +55,13 @@ def process(item)  # item is like "TextFile-abcd.xyz"
     return [ false, "Registered but cannot be moved" ]
   end
 
+  def successful_items
+    selected_items_from_messages_matching(/^\d+$/) # must be a numberic ID
+  end
+
+  def failed_items
+    selected_items_from_messages_matching(/^\D/) # Anything starting with not a digit
+  end
+
 end
 

BrainPortal/app/models/background_activity/register_file.rb

@@ -75,5 +75,13 @@ def process(item)  # item is like "TextFile-abcd.xyz"
     [ true, userfile.id ]
   end
 
+  def successful_items
+    selected_items_from_messages_matching(/^\d+$/) # must be a numberic ID
+  end
+
+  def failed_items
+    selected_items_from_messages_matching(/^\D/) # Anything starting with not a digit
+  end
+
 end