Skip to content

Commit 84f1a88

Browse files
pburkholderpburkholder
committed
Catch failures and count 'em
I non-prod I noticed that some of the tasks with --wait are failing. We don't want failing tasks to kill the entire job. We also don't want to bury failures. So this will keep track of the failues and mark the tasks as failing in CircleCI -- while still running as many as possible.
1 parent 3eaffd6 commit 84f1a88

File tree

1 file changed

+75
-71
lines changed

1 file changed

+75
-71
lines changed

‎.circleci/cron.sh‎

Lines changed: 75 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -4,79 +4,83 @@
44
# a non-zero exit code
55
set -e
66

7-
echo "Logging into cloud.gov"
7+
# Set # of failures to 0
8+
F=0
9+
10+
function run_production_tasks() {
11+
# === PRODUCTION environment ===================================================
12+
echo "Running tasks in Production..."
13+
14+
PROD_TASK="cf run-task touchpoints-production-sidekiq-worker --wait -c"
15+
# Users
16+
$PROD_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1))
17+
$PROD_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1))
18+
$PROD_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1))
19+
20+
# Forms
21+
# $PROD_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1))
22+
# $PROD_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1))
23+
# $PROD_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1))
24+
# $PROD_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1))
25+
$PROD_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1))
26+
# $PROD_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1))
27+
echo "Production tasks have completed."
28+
}
29+
30+
31+
function run_staging_tasks() {
32+
# === STAGING environment ======================================================
33+
echo "Running tasks in Staging..."
34+
35+
STAGING_TASK="cf run-task touchpoints-staging-sidekiq-worker --wait -c"
36+
37+
# Users
38+
$STAGING_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1))
39+
$STAGING_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1))
40+
$STAGING_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1))
41+
42+
# Forms
43+
$STAGING_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1))
44+
$STAGING_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1))
45+
$STAGING_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1))
46+
$STAGING_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1))
47+
$STAGING_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1))
48+
# $STAGING_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1))
49+
50+
echo "Staging tasks have completed."
51+
}
52+
53+
function run_demo_tasks() {
54+
# === DEMO environment =========================================================
55+
echo "Running tasks in Demo..."
56+
DEMO_TASK="cf run-task touchpoints-demo-sidekiq-worker --wait -c"
57+
58+
# Users
59+
$DEMO_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning" || F=$((F+=1))
60+
$DEMO_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning" || F=$((F+=1))
61+
$DEMO_TASK "rake scheduled_jobs:deactivate_inactive_users" || F=$((F+=1))
62+
63+
# Forms
64+
$DEMO_TASK "rake scheduled_jobs:send_daily_notifications" || F=$((F+=1))
65+
$DEMO_TASK "rake scheduled_jobs:send_weekly_notifications" || F=$((F+=1))
66+
$DEMO_TASK "rake scheduled_jobs:check_expiring_forms" || F=$((F+=1))
67+
$DEMO_TASK "rake scheduled_jobs:archive_forms" || F=$((F+=1))
68+
$DEMO_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms" || F=$((F+=1))
69+
# $DEMO_TASK "rake scheduled_jobs:delete_submissions_trash" || F=$((F+=1))
70+
71+
echo "Demo tasks have completed."
72+
}
73+
74+
echo "Logging into cloud.gov non-prod"
875
cf login -a $CF_API_ENDPOINT -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORG -s $CF_SPACE
9-
10-
#
11-
# === STAGING environment ======================================================
12-
#
13-
14-
echo "Running tasks in Staging..."
15-
16-
STAGING_TASK="cf run-task touchpoints-staging-sidekiq-worker --wait -c"
17-
18-
# Users
19-
$STAGING_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning"
20-
$STAGING_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning"
21-
$STAGING_TASK "rake scheduled_jobs:deactivate_inactive_users"
22-
23-
# Forms
24-
$STAGING_TASK "rake scheduled_jobs:send_daily_notifications"
25-
$STAGING_TASK "rake scheduled_jobs:send_weekly_notifications"
26-
$STAGING_TASK "rake scheduled_jobs:check_expiring_forms"
27-
$STAGING_TASK "rake scheduled_jobs:archive_forms"
28-
$STAGING_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms"
29-
# $STAGING_TASK "rake scheduled_jobs:delete_submissions_trash"
30-
31-
echo "Staging tasks have completed."
32-
33-
#
34-
# === DEMO environment =========================================================
35-
#
36-
37-
echo "Running tasks in Demo..."
38-
DEMO_TASK="cf run-task touchpoints-demo-sidekiq-worker --wait -c"
39-
40-
# Users
41-
$DEMO_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning"
42-
$DEMO_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning"
43-
$DEMO_TASK "rake scheduled_jobs:deactivate_inactive_users"
44-
45-
# Forms
46-
$DEMO_TASK "rake scheduled_jobs:send_daily_notifications"
47-
$DEMO_TASK "rake scheduled_jobs:send_weekly_notifications"
48-
$DEMO_TASK "rake scheduled_jobs:check_expiring_forms"
49-
$DEMO_TASK "rake scheduled_jobs:archive_forms"
50-
$DEMO_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms"
51-
# $DEMO_TASK "rake scheduled_jobs:delete_submissions_trash"
52-
53-
echo "Demo tasks have completed."
54-
76+
run_staging_tasks
77+
run_demo_tasks
5578
cf logout
5679

57-
#
58-
# === PRODUCTION environment ===================================================
59-
#
60-
61-
echo "Logging into cloud.gov"
80+
echo "Logging into cloud.gov production environment"
6281
cf login -a $CF_API_ENDPOINT -u $CF_PRODUCTION_SPACE_DEPLOYER_USERNAME -p $CF_PRODUCTION_SPACE_DEPLOYER_PASSWORD -o $CF_ORG -s prod
63-
64-
echo "Running tasks in Production..."
65-
66-
PROD_TASK="cf run-task touchpoints-production-sidekiq-worker --wait -c"
67-
# Users
68-
$PROD_TASK "rake scheduled_jobs:send_one_week_until_inactivation_warning"
69-
$PROD_TASK "rake scheduled_jobs:send_two_weeks_until_inactivation_warning"
70-
$PROD_TASK "rake scheduled_jobs:deactivate_inactive_users"
71-
72-
# Forms
73-
# $PROD_TASK "rake scheduled_jobs:send_daily_notifications"
74-
# $PROD_TASK "rake scheduled_jobs:send_weekly_notifications"
75-
# $PROD_TASK "rake scheduled_jobs:check_expiring_forms"
76-
# $PROD_TASK "rake scheduled_jobs:archive_forms"
77-
$PROD_TASK "rake scheduled_jobs:notify_form_managers_of_inactive_forms"
78-
# $PROD_TASK "rake scheduled_jobs:delete_submissions_trash"
79-
80-
echo "Production tasks have completed."
81-
82+
run_production_tasks
8283
cf logout
84+
85+
echo "$0 exiting with failure count: $F"
86+
exit $F

0 commit comments

Comments
 (0)