Skip to content

Commit fd8c264

Browse files
davidhooclaude
andcommitted
fix: recover zombie processing jobs and mark failed jobs properly
People background task could leave jobs stuck in "processing" status forever when processJob fails or writeGate deadlocks, because: 1. processJob error path never updated job status (only logged) 2. No runtime mechanism to detect stale processing jobs (only startup) 3. No panic recovery around writeGate.RLock areas Changes: - Add RecoverStaleProcessing to reset local processing jobs older than 30min back to queued, called in runBackground when no jobs are found - Add markJobFailed to set job=failed and reset photo face status on processJob error, preventing permanent "processing" state - Add panic recovery around writeGate.RLock/processJob and processPendingFaces to prevent RLock leaks on panic Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 04de8e3 commit fd8c264

2 files changed

Lines changed: 70 additions & 2 deletions

File tree

backend/internal/repository/people_job_repo.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ type PeopleJobRepository interface {
3434
HeartbeatRemote(id uint, workerID string, progress int, statusMsg string, lockUntil time.Time) error
3535
ReleaseRemote(id uint, workerID string, reason string, retryLater bool) error
3636
CompleteRemote(id uint, workerID string) error
37+
38+
// RecoverStaleProcessing resets local processing jobs that have been running
39+
// longer than timeout back to queued, so they can be reclaimed.
40+
RecoverStaleProcessing(timeout time.Duration) (int, error)
3741
}
3842

3943
type peopleJobRepository struct {
@@ -348,3 +352,21 @@ func (r *peopleJobRepository) CompleteRemote(id uint, workerID string) error {
348352
}
349353
return nil
350354
}
355+
356+
// RecoverStaleProcessing resets local processing jobs that have been running
357+
// longer than timeout back to queued, so they can be reclaimed by ClaimNextJob.
358+
func (r *peopleJobRepository) RecoverStaleProcessing(timeout time.Duration) (int, error) {
359+
cutoff := time.Now().Add(-timeout)
360+
result := r.db.Model(&model.PeopleJob{}).
361+
Where("status = ? AND (worker_id = ? OR worker_id IS NULL) AND started_at < ?",
362+
model.PeopleJobStatusProcessing, "", cutoff).
363+
Updates(map[string]interface{}{
364+
"status": model.PeopleJobStatusQueued,
365+
"started_at": nil,
366+
"status_message": "recovered from stale processing",
367+
})
368+
if result.Error != nil {
369+
return 0, result.Error
370+
}
371+
return int(result.RowsAffected), nil
372+
}

backend/internal/service/people_service.go

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -972,11 +972,19 @@ func (s *peopleService) runBackground(active *activePeopleTask) {
972972
fmt.Sprintf("正在处理照片 #%d", job.PhotoID), &job.PhotoID)
973973
s.setBackgroundBusy(true)
974974
s.writeGate.RLock()
975-
err = s.processJob(job)
975+
err = func() (retErr error) {
976+
defer func() {
977+
if r := recover(); r != nil {
978+
retErr = fmt.Errorf("processJob panic: %v", r)
979+
}
980+
}()
981+
return s.processJob(job)
982+
}()
976983
s.writeGate.RUnlock()
977984
s.setBackgroundBusy(false)
978985
if err != nil {
979986
s.appendBackgroundLog(fmt.Sprintf("处理人物任务 %d 失败:%v", job.ID, err))
987+
s.markJobFailed(job.ID, job.PhotoID, err.Error())
980988
}
981989

982990
s.taskMutex.Lock()
@@ -988,6 +996,15 @@ func (s *peopleService) runBackground(active *activePeopleTask) {
988996
continue
989997
}
990998

999+
// No detection job — recover stale processing jobs before clustering.
1000+
recovered, recoverErr := s.jobRepo.RecoverStaleProcessing(30 * time.Minute)
1001+
if recoverErr != nil {
1002+
s.appendBackgroundLog(fmt.Sprintf("恢复超时任务失败:%v", recoverErr))
1003+
} else if recovered > 0 {
1004+
s.appendBackgroundLog(fmt.Sprintf("已恢复 %d 个超时的检测任务", recovered))
1005+
continue // re-check ClaimNextJob immediately
1006+
}
1007+
9911008
// No detection job — check for processable pending faces and cluster.
9921009
// Use ListPending (same query as inner loop) to avoid mismatch with
9931010
// GetPendingStats which doesn't apply backoff filtering.
@@ -1023,7 +1040,14 @@ func (s *peopleService) runBackground(active *activePeopleTask) {
10231040

10241041
s.setBackgroundBusy(true)
10251042
s.writeGate.RLock()
1026-
hasMore, clusterErr := s.processPendingFaces()
1043+
hasMore, clusterErr := func() (hm bool, ce error) {
1044+
defer func() {
1045+
if r := recover(); r != nil {
1046+
ce = fmt.Errorf("processPendingFaces panic: %v", r)
1047+
}
1048+
}()
1049+
return s.processPendingFaces()
1050+
}()
10271051
s.writeGate.RUnlock()
10281052
s.setBackgroundBusy(false)
10291053
if clusterErr != nil {
@@ -1053,6 +1077,28 @@ func (s *peopleService) runBackground(active *activePeopleTask) {
10531077
}
10541078

10551079
}
1080+
1081+
// markJobFailed marks a processing job as failed and resets the photo's
1082+
// face_process_status so it can be re-enqueued later.
1083+
func (s *peopleService) markJobFailed(jobID uint, photoID uint, errMsg string) {
1084+
now := time.Now()
1085+
s.executeWrite(func() error {
1086+
return s.db.Transaction(func(tx *gorm.DB) error {
1087+
if err := tx.Model(&model.PeopleJob{}).Where("id = ? AND status = ?", jobID, model.PeopleJobStatusProcessing).
1088+
Updates(map[string]interface{}{
1089+
"status": model.PeopleJobStatusFailed,
1090+
"last_error": errMsg,
1091+
"completed_at": &now,
1092+
}).Error; err != nil {
1093+
return err
1094+
}
1095+
return tx.Model(&model.Photo{}).Where("id = ? AND face_process_status IN ?",
1096+
photoID, []string{model.FaceProcessStatusPending, model.FaceProcessStatusProcessing}).
1097+
Update("face_process_status", model.FaceProcessStatusNone).Error
1098+
})
1099+
})
1100+
}
1101+
10561102
func (s *peopleService) processJob(job *model.PeopleJob) error {
10571103
photo, skip, err := s.preflightCheck(job)
10581104
if err != nil {

0 commit comments

Comments
 (0)