Skip to content

Commit

Permalink
Small changes.
Browse files Browse the repository at this point in the history 
1. BourreauWorker logs contains type of task and task ID
2. New scopes for tasks (on_cpu etc)
3. Random passwords always start with 'T'
  • Loading branch information
@prioux
prioux committed Mar 11, 2024
1 parent 7c68e84 commit 0e28364
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 21 deletions.
38 changes: 19 additions & 19 deletions Bourreau/app/models/bourreau_worker.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,13 +275,13 @@ def process_task(task) # when entering this methods task is a partial object, wi
initial_status = task.status
initial_change_time = task.updated_at

worker_log.debug "--- Got #{task.bname_tid} in state #{initial_status}"
worker_log.debug "--- Got #{task.tname_tid} in state #{initial_status}"

unless task.status =~ /^(Recover|Restart)/
task.update_status
new_status = task.status

worker_log.debug "Updated #{task.bname_tid} to state #{new_status}"
worker_log.debug "Updated #{task.tname_tid} to state #{new_status}"

# Mechanism for tasks to submit other tasks: tasks that are active
# may submit new tasks dynamically provided that they have the :can_submit_new_tasks
Expand Down Expand Up @@ -325,17 +325,17 @@ def process_task(task) # when entering this methods task is a partial object, wi
# We need to raise an exception if we cannot successfully
# transition ourselves.
task.status_transition!("New","Setting Up")
worker_log.debug "Start #{task.bname_tid}"
worker_log.info "Setup #{task.tname_tid}"
task.addlog_current_resource_revision
task.addlog_context(self,"#{self.pretty_name}")
task.setup_and_submit_job # New -> Queued|Failed To Setup
worker_log.info "Submitted: #{task.bname_tid}"
worker_log.debug " -> #{task.bname_tid} to state #{task.status}"
worker_log.info "Submitted: #{task.tname_tid} as #{task.cluster_jobid}"
worker_log.debug " -> #{task.tname_tid} to state #{task.status}"
task.meta[:last_delay_new_to_queued] = (Time.now - initial_change_time).to_i
elsif action == :wait
worker_log.debug " -> #{task.bname_tid} unfulfilled Setup prerequisites."
worker_log.debug " -> #{task.tname_tid} unfulfilled Setup prerequisites."
else # action == :fail
worker_log.debug " -> #{task.bname_tid} failed Setup prerequisites."
worker_log.debug " -> #{task.tname_tid} failed Setup prerequisites."
task.status_transition(task.status, "Failed Setup Prerequisites")
task.addlog_context(self,"#{self.pretty_name} detected failed Setup prerequisites")
task.save
Expand All @@ -351,16 +351,16 @@ def process_task(task) # when entering this methods task is a partial object, wi
# We need to raise an exception if we cannot successfully
# transition ourselves.
task.status_transition!("Data Ready","Post Processing")
worker_log.debug "PostPro #{task.bname_tid}"
worker_log.debug "PostPro #{task.tname_tid}"
task.addlog_current_resource_revision
task.addlog_context(self,"#{self.pretty_name}")
task.post_process # Data Ready -> Completed|Failed To PostProcess|Failed On Cluster
worker_log.info "PostProcess: #{task.bname_tid}"
worker_log.debug " -> #{task.bname_tid} to state #{task.status}"
worker_log.info "PostProcess: #{task.tname_tid}"
worker_log.debug " -> #{task.tname_tid} to state #{task.status}"
elsif action == :wait
worker_log.debug " -> #{task.bname_tid} unfulfilled PostProcessing prerequisites."
worker_log.debug " -> #{task.tname_tid} unfulfilled PostProcessing prerequisites."
else # action == :fail
worker_log.debug " -> #{task.bname_tid} failed PostProcessing prerequisites."
worker_log.debug " -> #{task.tname_tid} failed PostProcessing prerequisites."
task.status_transition(task.status, "Failed PostProcess Prerequisites")
task.addlog_context(self,"#{self.pretty_name} detected failed PostProcessing prerequisites")
task.save
Expand Down Expand Up @@ -505,7 +505,7 @@ def process_task(task) # when entering this methods task is a partial object, wi
:message_type => :notice,
:header => "Task #{task.name} Completed Successfully",
:description => "Oh great!",
:variable_text => "[[#{task.bname_tid}][/tasks/#{task.id}]]"
:variable_text => "[[#{task.fullname}][/tasks/#{task.id}]]"
)
elsif task.status =~ /^Failed/
Message.send_message(task.user,
Expand All @@ -514,7 +514,7 @@ def process_task(task) # when entering this methods task is a partial object, wi
:description => "Sorry about that. Check the task's log.\n" +
"Consider using the 'Recovery' button to try the task again, in case\n" +
"it's just a system error: CBRAIN will do its best to fix it.",
:variable_text => "[[#{task.bname_tid}][/tasks/#{task.id}]]"
:variable_text => "[[#{task.fullname}][/tasks/#{task.id}]]"
)
end
end
Expand All @@ -523,17 +523,17 @@ def process_task(task) # when entering this methods task is a partial object, wi
# setup_and_submit_job() or post_process(); it's allowed, it means
# some other worker has beaten us to the punch. So we just ignore it.
rescue CbrainTransitionException => te
worker_log.debug "Transition Exception: task '#{task.bname_tid}' FROM='#{te.from_state}' TO='#{te.to_state}' FOUND='#{te.found_state}'"
worker_log.debug "Transition Exception: task '#{task.tname_tid}' FROM='#{te.from_state}' TO='#{te.to_state}' FOUND='#{te.found_state}'"
return

# Our task might have disappeared. Rare.
# This is most likely during the reload() at the top of the method.
rescue ActiveRecord::RecordNotFound => gone_ex
if gone_ex.message["#{task.class}"] && gone_ex.message =~ /=#{task.id}\b/ # our own task?
worker_log.debug "Task '#{task.bname_tid}' has disappeared. Skipping."
worker_log.debug "Task '#{task.tname_tid}' has disappeared. Skipping."
return # nothing else to do
else
worker_log.fatal "Some object has disappeared while processing task '#{task.bname_tid}': #{gone_ex.class.to_s} #{gone_ex.message}\n" + gone_ex.backtrace[0..10].join("\n")
worker_log.fatal "Some object has disappeared while processing task '#{task.tname_tid}': #{gone_ex.class.to_s} #{gone_ex.message}\n" + gone_ex.backtrace[0..10].join("\n")
raise gone_ex # something ELSE than a task disappeared?
end

Expand All @@ -542,7 +542,7 @@ def process_task(task) # when entering this methods task is a partial object, wi
# so if an exception went through anyway, it's a CODING BUG
# in this worker's logic.
rescue => e
worker_log.fatal "Exception processing task #{task.bname_tid}: #{e.class.to_s} #{e.message}\n" + e.backtrace[0..10].join("\n")
worker_log.fatal "Exception processing task #{task.tname_tid}: #{e.class.to_s} #{e.message}\n" + e.backtrace[0..10].join("\n")
raise e

end # process_task which is way too long.
Expand All @@ -556,7 +556,7 @@ def check_for_tasks_stuck_in_ruby
task.mark_as_failed_in_ruby rescue nil
if task.status != orig_status
task.addlog("Worker detects that task is too old and stuck at '#{orig_status}'; status reset to '#{task.status}'")
worker_log.info "Stuck: #{task.bname_tid} from #{orig_status} to state #{task.status}"
worker_log.info "Stuck: #{task.tname_tid} from #{orig_status} to state #{task.status}"
end
end
end
Expand Down
3 changes: 3 additions & 0 deletions BrainPortal/app/models/cbrain_task.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ class CbrainTask < ApplicationRecord
scope :failed_setup, -> { where( "cbrain_tasks.status" => 'Failed To Setup' ) }
scope :failed_cluster, -> { where( "cbrain_tasks.status" => 'Failed On Cluster' ) }
scope :failed_post, -> { where( "cbrain_tasks.status" => 'Failed To PostProcess' ) }
scope :setting_up, -> { where( "cbrain_tasks.status" => 'Setting Up' ) }
scope :on_cpu, -> { where( "cbrain_tasks.status" => 'On CPU' ) }
scope :post_proc, -> { where( "cbrain_tasks.status" => 'Post Processing' ) }
scope :completed, -> { where( "cbrain_tasks.status" => 'Completed' ) }

scope :real_tasks,
Expand Down
4 changes: 2 additions & 2 deletions BrainPortal/app/models/user.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,8 +527,8 @@ def password_required? #:nodoc:

# Create a random string (currently for passwords).
def self.random_string
length = rand(5) + 4
s = ""
length = rand(4) + 8
s = "T"
length.times do
c = rand(75) + 48 # ascii range from '0' to 'z'
redo if c == 92 || c == 96 # \ or '
Expand Down

0 comments on commit 0e28364

Please sign in to comment.