Merge branch 'master' into patch-nginxlogrotate

hieradata/common.yaml

@@ -5,8 +5,8 @@ puppetserver: false
 puppet_major_version: 8
 
 mediawiki::multiversion::versions:
-  '1.42':
-    branch: 'REL1_42'
+  '1.43':
+    branch: 'REL1_43'
     default: true
 
 puppetserver_hostname: puppet181.wikitide.net
@@ -17,7 +17,6 @@ mediawiki::is_canary: false
 mediawiki::default_sync: 'all'
 mediawiki::use_shellbox: false
 mediawiki::jobqueue::wiki: 'loginwiki'
-mediawiki::jobqueue::runner::beta: false
 mediawiki::monitoring::host: 'login.miraheze.org'
 memcached_servers:
   - '10.0.15.113:11211:1 "shard01"'
@@ -67,6 +66,46 @@ varnish::backends:
     probe: mwhealth
     pool: true
     xdebug: true
+  mw153:
+    port: 8121
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw154:
+    port: 8122
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw163:
+    port: 8123
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw164:
+    port: 8124
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw173:
+    port: 8125
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw174:
+    port: 8126
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw183:
+    port: 8127
+    probe: mwhealth
+    pool: true
+    xdebug: true
+  mw184:
+    port: 8128
+    probe: mwhealth
+    pool: true
+    xdebug: true
   mwtask171:
     port: 8161
     probe: false
@@ -77,6 +116,16 @@ varnish::backends:
     probe: false
     pool: false
     xdebug: true
+  mwtask151:
+    port: 8162
+    probe: false
+    pool: false
+    xdebug: true
+  mwtask161:
+    port: 8163
+    probe: false
+    pool: false
+    xdebug: true
   test151:
     port: 8181
     probe: false

hieradata/hosts/changeprop151.yaml

@@ -1,39 +1,68 @@
 changeprop::jobqueue: true
-changeprop::jobrunner_host: 'http://10.0.17.144:9006'
-changeprop::import_host: 'http://10.0.17.144:9006'
-changeprop::videoscaler_host: 'http://10.0.17.144:9006'
-changeprop::num_workers: 12
-changeprop::global_new_files_concurrency: 25
-changeprop::semantic_mediawiki_concurrency: 50
+changeprop::jobrunner_host: 'http://localhost:9007'
+changeprop::jobrunner_high_timeout_host: 'http://localhost:9008'
+changeprop::videoscaler_host: 'http://localhost:9009'
+jobrunner_haproxy::backends:
+  mwtask151:
+    ip: 10.0.15.150
+    port: 9006
+  mwtask161:
+    ip: 10.0.16.157
+    port: 9006
+  mwtask171:
+    ip: 10.0.17.144
+    port: 9006
+  mwtask181:
+    ip: 10.0.18.106
+    port: 9006
+changeprop::num_workers: 8
+changeprop::semantic_mediawiki_concurrency: 2
 changeprop::low_traffic_concurrency: 50
 changeprop::high_traffic_jobs_config:
   ThumbnailRender:
     concurrency: 5
   categoryMembershipChange:
-    concurrency: 50
+    concurrency: 10
   # CNDPurge is quite low-volume, but it uses delayed execution,
   # so avoid putting it together with other low-volume jobs so that it doesn't
   # block execution for others.
   cdnPurge:
-    concurrency: 20
+    concurrency: 10
   # RecordLinks is normally low-volume, but could have big spikes
   # when maintenance scripts are run. Elevated concurrency
   RecordLintJob:
-    concurrency: 25
+    concurrency: 15
     consumer_batch_size: 10
   wikibase-addUsagesForPage:
-    concurrency: 5
+    concurrency: 4
   LocalGlobalUserPageCacheUpdateJob:
     # This job is prone to large spikes, so having it on the low_traffic_jobs queue
     # blocks other jobs.
-    concurrency: 10
+    concurrency: 15
+  notificationGetStartedJob:
+    concurrency: 2
+    # All the jobs of this kind are delayed exactly 48 hours, avoid reenqueueing them
+    # by setting the reenqueue delay to 72 hours
+    reenqueue_delay: 259200
+  notificationKeepGoingJob:
+    concurrency: 2
+    # All the jobs of this kind are delayed exactly 48 hours, avoid reenqueueing them
+    # by setting the reenqueue delay to 72 hours
+    reenqueue_delay: 259200
+  newcomerTasksCacheRefreshJob:
+    concurrency: 2
+    # All the jobs of this kind are delayed exactly 144 hours (6 days), avoid reenqueueing them
+    # by setting the reenqueue delay to 7 days
+    reenqueue_delay: 604800
+  refreshUserImpactJob:
+    concurrency: 3
   # For cirrus search jobs the retries are built into the job itself,
   # so disable the retries by change-prop. We need special rules for cirrus
   # jobs because they need special configuration.
   cirrusSearchCheckerJob:
     disable_delayed_execution: true
     retry_limit: 0
-    concurrency: 20
+    concurrency: 15
   cirrusSearchDeleteArchive:
     retry_limit: 0
     concurrency: 5
@@ -45,82 +74,119 @@ changeprop::high_traffic_jobs_config:
     concurrency: 15
   cirrusSearchLinksUpdate:
     retry_limit: 0
-    concurrency: 100
+    concurrency: 15
   cirrusSearchLinksUpdatePrioritized:
     retry_limit: 0
-    concurrency: 50
+    concurrency: 10
   cirrusSearchOtherIndex:
     retry_limit: 0
     concurrency: 5
-  cirrusSearchElasticaWrite:
-    retry_limit: 0
-    concurrency: 25
-    reenqueue_delay: 3600
+  wikibase-InjectRCRecords:
+    concurrency: 2
   parsoidCachePrewarm:
-    concurrency: 150
+    concurrency: 15
   htmlCacheUpdate:
-    concurrency: 50
-    # Abandon jobs which root job is more than 1 week long
-    root_claim_ttl: 604800
+    concurrency: 10
   refreshLinks:
-    concurrency: 25
+    concurrency: 15
     # Abandon jobs which root job is more than 1 week long
     root_claim_ttl: 604800
   smw.changePropagationClassUpdate:
-    concurrency: 50
+    concurrency: 2
   smw.changePropagationDispatch:
-    concurrency: 10
+    concurrency: 2
+  smw.changePropagationUpdate:
+    concurrency: 2
   # Translation jobs tend to be low traffic but are being delayed when other
   # low traffic jobs have a large spike. It is being moved to its own queue to
   # improve editing experience for users
   UpdateTranslatablePageJob:
-    concurrency: 3
+    concurrency: 2
   RenderTranslationPageJob:
-    concurrency: 3
+    concurrency: 2
+  MWScriptJob:
+    concurrency: 2
 
   # These jobs need to be ran with priority so are using their own queues
-  CreateWikiJob:
+  LocalRenameUserJob:
     disable_delayed_execution: true
     concurrency: 1
-  RemovePIIJob:
+  SetContainersAccessJob:
+    disable_delayed_execution: true
+    concurrency: 2
+  CreateWikiJob:
     disable_delayed_execution: true
     concurrency: 1
+  ImportDumpNotifyJob:
+    disable_delayed_execution: true
+    concurrency: 2
+  RottenLinksJob:
+    concurrency: 6
+  GlobalNewFilesDeleteJob:
+    concurrency: 2
+  GlobalNewFilesInsertJob:
+    concurrency: 2
+  GlobalNewFilesMoveJob:
+    concurrency: 2
 
-changeprop::import_jobs_config:
+changeprop::high_traffic_high_timeout_jobs_config:
+  cirrusSearchElasticaWrite:
+    retry_limit: 0
+    concurrency: 15
+    reenqueue_delay: 3600
+    timeout: 600000
+  NamespaceMigrationJob:
+    disable_delayed_execution: true
+    concurrency: 1
+    timeout: 9000000
   DataDumpGenerateJob:
     disable_delayed_execution: true
     concurrency: 1
+    timeout: 86400000
   ImportDumpJob:
     disable_delayed_execution: true
     retry_limit: 0
     concurrency: 1
-  ImportDumpNotifyJob:
+    timeout: 259200000
+  RemovePIIJob:
     disable_delayed_execution: true
     concurrency: 1
+    timeout: 86400000
   RequestWikiAIJob:
     concurrency: 1
-  RottenLinksJob:
-    concurrency: 1
+    timeout: 86400000
 
 changeprop::videoscaler_jobs_config:
   webVideoTranscode:
-    timeout: 86400
-    concurrency: 3
+    timeout: 86400000
+    concurrency: 1
+    retry_limit: 1
+    consumer:
+      max.poll.interval.ms: 3600000  # 1h
   webVideoTranscodePrioritized:
-    concurrency: 3
-    timeout: 86400
+    concurrency: 1
+    timeout: 86400000
+    retry_limit: 1
+    consumer:
+      max.poll.interval.ms: 3600000  # 1h
 
-changeprop::global_new_files_jobs:
-  - GlobalNewFilesDeleteJob
-  - GlobalNewFilesInsertJob
-  - GlobalNewFilesMoveJob
+changeprop::latency_sensitive_jobs_config:
+  # AssembleUploadChunks, PublishStashedFile, and UploadFromUrl,
+  # are considered latency sensitive, as high backlog time presents a poor
+  # user experience for async uploads.
+  AssembleUploadChunks:
+    concurrency: 5
+  PublishStashedFile:
+    concurrency: 5
+  UploadFromUrl:
+    concurrency: 5
 
 changeprop::semantic_mediawiki_jobs:
   - SMWRefreshJob
   - SMWUpdateJob
 #  - smw.changePropagationClassUpdate
 #  - smw.changePropagationDispatch
-  - smw.changePropagationUpdate
+#  - smw.changePropagationUpdate
   - smw.deferredConstraintCheckUpdateJob
   - smw.elasticFileIngest
   - smw.elasticIndexerRecovery
@@ -211,3 +277,6 @@ role::prometheus::statsd_exporter::mappings:
       partition: $3
 
 http_proxy: 'http://bastion.wikitide.net:8080'
+
+redis::heap: '7GB'
+redis::maxmemory_policy: volatile-lru

hieradata/hosts/cp36.yaml

@@ -1,6 +1,7 @@
 users::groups: 'cache-admins'
 varnish::cache_file_size: '60g'
 varnish::thread_pool_max: 12000
+varnish::transient_storage: 1
 nginx::worker_processes: 6
 
 base::syslog::rsyslog_udp_localhost: true

hieradata/hosts/cp37.yaml

@@ -1,7 +1,10 @@
 users::groups: 'cache-admins'
-varnish::cache_file_size: '60g'
+varnish::cache_file_size: '380G'
 varnish::thread_pool_max: 12000
-nginx::worker_processes: 6
+varnish::transient_storage: 5
+nginx::worker_processes: 8
+role::varnish::restrict_firewall: true
+nginx::use_varnish_directly: false
 
 base::syslog::rsyslog_udp_localhost: true
 nginx::logrotate_number: 2

hieradata/hosts/db151.yaml

@@ -1,7 +1,9 @@
 users::groups: 'database-admins'
 http_proxy: 'http://bastion.wikitide.net:8080'
-role::db::weekly_misc: ['mhglobal']
-mariadb::config::innodb_buffer_pool_size: '30G'
+role::db::weekly_misc: []
+# This is in GiB
+mariadb::config::innodb_buffer_pool_size: '34G'
 mariadb::version: '10.11'
 role::db::enable_ssl: false
 mariadb::config::server_id: 5
+role::db::is_beta_db: false