Skip to content

Commit 0eb5ae0

Browse files
jcscottiiijcscottiii
authored
infra: increase scheduler frequency to 6-hour intervals with staggered execution (#2214)
Updates Cloud Scheduler cron configurations for both prod and staging to run data pipelines more frequently (4x daily) while preventing execution overlaps. Changes include: - Increased frequency: Jobs now run every 6 hours (cycles starting at 0, 6, 12, 18). - Regional staggering: `us-central1` runs at minute 0, `europe-west1` runs at minute 30 to prevent simultaneous resource contention. - Defined execution order to respect data dependencies: 1. BCD (Start of cycle) 2. Web Features (T+1 hour) 3. Developer Signals & Chromium (T+2 hours) 4. UMA & Web Features Mapping (T+3 hours) - WPT schedules remain unchanged (running once daily).
1 parent b1736d1 commit 0eb5ae0

File tree

2 files changed

+54
-29
lines changed

2 files changed

+54
-29
lines changed

‎infra/.envs/prod.tfvars‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,41 +53,53 @@ backend_cache_settings = {
5353

5454
backend_cors_allowed_origin = "https://*"
5555

56+
# 1. BCD - The Start
57+
# US runs at minute 00, EU runs at minute 30
5658
bcd_region_schedules = {
57-
"us-central1" = "0 19 * * *" # Daily at 7:00 PM
58-
"europe-west1" = "0 7 * * *" # Daily at 7:00 AM
59+
"us-central1" = "0 0,6,12,18 * * *"
60+
"europe-west1" = "30 0,6,12,18 * * *"
5961
}
6062

63+
# 2. Web Features - 1 Hour Later
64+
# Needs the BCD data first for Browser Releases
65+
# (Hours shifted to 1, 7, 13, 19)
6166
web_features_region_schedules = {
62-
"us-central1" = "0 20 * * *" # Daily at 8:00 PM
63-
"europe-west1" = "0 8 * * *" # Daily at 8:00 AM
67+
"us-central1" = "0 1,7,13,19 * * *"
68+
"europe-west1" = "30 1,7,13,19 * * *"
6469
}
6570

71+
# 3. Signals / Chromium / Web Features Mapping - 2 Hours Later
72+
# These need the Web Features data first
73+
# (Hours shifted to 2, 8, 14, 20)
6674
developer_signals_region_schedules = {
67-
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
68-
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
75+
"us-central1" = "0 2,8,14,20 * * *"
76+
"europe-west1" = "30 2,8,14,20 * * *"
6977
}
7078

7179
chromium_region_schedules = {
72-
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
73-
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
80+
"us-central1" = "0 2,8,14,20 * * *"
81+
"europe-west1" = "30 2,8,14,20 * * *"
82+
}
83+
84+
web_features_mapping_region_schedules = {
85+
"us-central1" = "0 3,9,15,21 * * *"
86+
"europe-west1" = "30 3,9,15,21 * * *"
7487
}
7588

89+
# 4. UMA - 3 Hours Later
90+
# UMA needs the Chromium data first
91+
# (Hours shifted to 3, 9, 15, 21)
7692
uma_region_schedules = {
77-
"us-central1" = "0 22 * * *" # Daily at 10:00 PM
78-
"europe-west1" = "0 10 * * *" # Daily at 10:00 AM
93+
"us-central1" = "0 3,9,15,21 * * *"
94+
"europe-west1" = "30 3,9,15,21 * * *"
7995
}
8096

97+
# 5. WPT - Takes a while. It runs once daily.
8198
wpt_region_schedules = {
8299
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
83100
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
84101
}
85102

86-
web_features_mapping_region_schedules = {
87-
"us-central1" = "0 23 * * *" # Daily at 11:00 PM
88-
"europe-west1" = "0 11 * * *" # Daily at 11:00 AM
89-
}
90-
91103
firebase_api_key_location = "prod-firebase-app-api-key"
92104

93105
auth_github_config_locations = {

‎infra/.envs/staging.tfvars‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -54,40 +54,53 @@ backend_cache_settings = {
5454
# Needed for UbP.
5555
backend_cors_allowed_origin = "https://website-webstatus-dev.corp.goog"
5656

57+
# 1. BCD - The Start
58+
# US runs at minute 00, EU runs at minute 30
5759
bcd_region_schedules = {
58-
"us-central1" = "0 19 * * *" # Daily at 7:00 PM
59-
"europe-west1" = "0 7 * * *" # Daily at 7:00 AM
60+
"us-central1" = "0 0,6,12,18 * * *"
61+
"europe-west1" = "30 0,6,12,18 * * *"
6062
}
6163

64+
# 2. Web Features - 1 Hour Later
65+
# Needs the BCD data first for Browser Releases
66+
# (Hours shifted to 1, 7, 13, 19)
6267
web_features_region_schedules = {
63-
"us-central1" = "0 20 * * *" # Daily at 8:00 PM
64-
"europe-west1" = "0 8 * * *" # Daily at 8:00 AM
68+
"us-central1" = "0 1,7,13,19 * * *"
69+
"europe-west1" = "30 1,7,13,19 * * *"
6570
}
6671

72+
# 3. Signals / Chromium / Web Features Mapping - 2 Hours Later
73+
# These need the Web Features data first
74+
# (Hours shifted to 2, 8, 14, 20)
6775
developer_signals_region_schedules = {
68-
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
69-
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
76+
"us-central1" = "0 2,8,14,20 * * *"
77+
"europe-west1" = "30 2,8,14,20 * * *"
7078
}
7179

7280
chromium_region_schedules = {
73-
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
74-
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
81+
"us-central1" = "0 2,8,14,20 * * *"
82+
"europe-west1" = "30 2,8,14,20 * * *"
7583
}
7684

85+
web_features_mapping_region_schedules = {
86+
"us-central1" = "0 3,9,15,21 * * *"
87+
"europe-west1" = "30 3,9,15,21 * * *"
88+
}
89+
90+
# 4. UMA - 3 Hours Later
91+
# UMA needs the Chromium data first
92+
# (Hours shifted to 3, 9, 15, 21)
7793
uma_region_schedules = {
78-
"us-central1" = "0 22 * * *" # Daily at 10:00 PM
79-
"europe-west1" = "0 10 * * *" # Daily at 10:00 AM
94+
"us-central1" = "0 3,9,15,21 * * *"
95+
"europe-west1" = "30 3,9,15,21 * * *"
8096
}
8197

98+
# 5. WPT - Takes a while. It runs once daily.
8299
wpt_region_schedules = {
83100
"us-central1" = "0 21 * * *" # Daily at 9:00 PM
84101
"europe-west1" = "0 9 * * *" # Daily at 9:00 AM
85102
}
86103

87-
web_features_mapping_region_schedules = {
88-
"us-central1" = "0 23 * * *" # Daily at 11:00 PM
89-
"europe-west1" = "0 11 * * *" # Daily at 11:00 AM
90-
}
91104

92105
firebase_api_key_location = "staging-firebase-app-api-key"
93106

0 commit comments

Comments
 (0)