Named params and temporal drift

EbTech · EbTech · commit 82c2a1ccd310 · 2023-10-23T00:04:31.000-07:00
diff --git a/experiments/testing/codechef-init.json b/experiments/testing/codechef-init.json
@@ -1,11 +1,17 @@
 {
-  "save_checkpoint": "../experiments/testing/codechef-state.json",
-  "max_contests": 630,
-  "mu_noob": 1600,
-  "sig_noob": 280,
-  "contest_source": "codechef",
-  "system": {
-    "method": "mmr-simple",
-    "params": [0.2, 80, 1, 50, 0.1]
-  }
-}
+    "save_checkpoint": "../experiments/testing/codechef-state.json",
+    "max_contests": 630,
+    "mu_noob": 1600,
+    "sig_noob": 280,
+    "contest_source": "codechef",
+    "system": {
+        "method": "mmr-simple",
+        "weight_limit": 0.2,
+        "noob_delay": [0.6, 0.8],
+        "sig_limit": 80,
+        "drift_per_day": 0,
+        "split_ties": true,
+        "history_len": 50,
+        "transfer_speed": 0.1
+    }
+}
diff --git a/experiments/testing/codechef-update.json b/experiments/testing/codechef-update.json
@@ -1,13 +1,19 @@
 {
-  "load_checkpoint": "../experiments/testing/codechef-state.json",
-  "save_checkpoint": "../experiments/testing/codechef-state.json",
-  "skip_contests": 630,
-  "max_contests": 999,
-  "mu_noob": 1500,
-  "sig_noob": 280,
-  "contest_source": "codechef",
-  "system": {
-    "method": "mmr-simple",
-    "params": [0.2, 80, 1, 50, 0.1]
-  }
-}
+    "load_checkpoint": "../experiments/testing/codechef-state.json",
+    "save_checkpoint": "../experiments/testing/codechef-state.json",
+    "skip_contests": 630,
+    "max_contests": 999,
+    "mu_noob": 1500,
+    "sig_noob": 280,
+    "contest_source": "codechef",
+    "system": {
+        "method": "mmr-simple",
+        "weight_limit": 0.2,
+        "noob_delay": [0.6, 0.8],
+        "sig_limit": 80,
+        "drift_per_day": 0,
+        "split_ties": true,
+        "history_len": 50,
+        "transfer_speed": 0.1
+    }
+}
diff --git a/experiments/testing/mmr-iowa-runners.json b/experiments/testing/mmr-iowa-runners.json
@@ -0,0 +1,16 @@
+{
+    "max_contests": 9999,
+    "mu_noob": 1500,
+    "sig_noob": 350,
+    "contest_source": "iaxcindividuals",
+    "system": {
+        "method": "mmr-simple",
+        "weight_limit": 0.2,
+        "noob_delay": [],
+        "sig_limit": 0,
+        "drift_per_day": 50,
+        "split_ties": false,
+        "history_len": 999,
+        "transfer_speed": 0.1
+    }
+}
diff --git a/multi-skill/src/experiment_config.rs b/multi-skill/src/experiment_config.rs
@@ -15,15 +15,41 @@ use std::path::Path;
 #[derive(Deserialize, Debug)]
 #[serde(tag = "method", rename_all = "kebab-case")]
 pub enum SystemParams {
-    Glicko { params: Vec<f64> },
-    Bar { params: Vec<f64> },
-    Endure { params: Vec<f64> },
-    Cfsys { params: Vec<f64> },
-    Tcsys { params: Vec<f64> },
-    Trueskill { params: Vec<f64> },
-    Mmx { params: Vec<f64> },
-    Mmr { params: Vec<f64> },
-    MmrSimple { params: Vec<f64> },
+    Glicko {
+        params: Vec<f64>,
+    },
+    Bar {
+        params: Vec<f64>,
+    },
+    Endure {
+        params: Vec<f64>,
+    },
+    Cfsys {
+        params: Vec<f64>,
+    },
+    Tcsys {
+        params: Vec<f64>,
+    },
+    Trueskill {
+        params: Vec<f64>,
+    },
+    Mmx {
+        params: Vec<f64>,
+    },
+    Mmr {
+        params: Vec<f64>,
+    },
+    // Experimental support for named config params
+    // TODO: find nice ways to use defaults and interface with constructor
+    MmrSimple {
+        weight_limit: f64,
+        noob_delay: Vec<f64>,
+        sig_limit: f64,
+        drift_per_day: f64,
+        split_ties: bool,
+        history_len: usize,
+        transfer_speed: f64,
+    },
 }
 
 fn usize_zero() -> usize {
@@ -119,7 +145,7 @@ impl Experiment {
                 weight_limit: params[0],
                 noob_delay: vec![], // TODO: add this to the config spec
                 sig_limit: params[1],
-                drift_per_sec: 0.,
+                drift_per_day: 0.,
                 split_ties: params[2] > 0.,
                 subsample_size: params[3] as usize,
                 subsample_bucket: params[4],
@@ -129,20 +155,28 @@ impl Experiment {
                 weight_limit: params[0],
                 noob_delay: vec![], // TODO: add this to the config spec
                 sig_limit: params[1],
-                drift_per_sec: 0.,
+                drift_per_day: 0.,
                 split_ties: params[2] > 0.,
                 subsample_size: params[3] as usize,
                 subsample_bucket: params[4],
                 variant: EloMMRVariant::Logistic(params[5]),
             }),
-            SystemParams::MmrSimple { params } => Box::new(SimpleEloMMR {
-                weight_limit: params[0],
-                noob_delay: vec![0.6, 0.8], // TODO: add this to the config spec
-                sig_limit: params[1],
-                drift_per_sec: 0.,
-                split_ties: params[2] > 0.,
-                history_len: params[3] as usize,
-                transfer_speed: params[4],
+            SystemParams::MmrSimple {
+                weight_limit,
+                noob_delay,
+                sig_limit,
+                drift_per_day,
+                split_ties,
+                history_len,
+                transfer_speed,
+            } => Box::new(SimpleEloMMR {
+                weight_limit,
+                noob_delay,
+                sig_limit,
+                drift_per_day,
+                split_ties,
+                history_len,
+                transfer_speed,
             }),
         };
 
diff --git a/multi-skill/src/systems/elo_mmr.rs b/multi-skill/src/systems/elo_mmr.rs
@@ -1,5 +1,5 @@
 //! Elo-R system details: https://arxiv.org/abs/2101.00400
-use super::{Player, Rating, RatingSystem, TanhTerm};
+use super::{Player, Rating, RatingSystem, TanhTerm, SECS_PER_DAY};
 use crate::data_processing::ContestRatingParams;
 use crate::numerical::{solve_newton, standard_normal_cdf, standard_normal_pdf};
 use core::ops::Range;
@@ -146,8 +146,8 @@ pub struct EloMMR {
     // each contest participation adds an amount of drift such that, in the absence of
     // much time passing, the limiting skill uncertainty's square approaches this value
     pub sig_limit: f64,
-    // additional variance per second, from a drift that's continuous in time
-    pub drift_per_sec: f64,
+    // additional variance per day, from a drift that's continuous in time
+    pub drift_per_day: f64,
     // whether to count ties as half a win plus half a loss
     pub split_ties: bool,
     // maximum number of opponents and recent events to use, as a compute-saving approximation
@@ -195,20 +195,32 @@ impl EloMMR {
             weight_limit,
             noob_delay,
             sig_limit,
-            drift_per_sec: 0.,
+            drift_per_day: 0.,
             split_ties,
             subsample_size,
             subsample_bucket,
             variant,
         }
     }
 
-    fn sig_perf_and_drift(&self, mut contest_weight: f64, n: usize) -> (f64, f64) {
+    fn compute_weight(&self, mut contest_weight: f64, n: usize) -> f64 {
         contest_weight *= self.weight_limit;
-        contest_weight *= self.noob_delay.get(n).unwrap_or(&1.);
-        let sig_perf = (1. + 1. / contest_weight).sqrt() * self.sig_limit;
-        let sig_drift_sq = contest_weight * self.sig_limit * self.sig_limit;
-        (sig_perf, sig_drift_sq)
+        if let Some(delay_factor) = self.noob_delay.get(n) {
+            contest_weight *= delay_factor;
+        }
+        contest_weight
+    }
+
+    fn compute_sig_perf(&self, weight: f64) -> f64 {
+        let discrete_perf = (1. + 1. / weight) * self.sig_limit * self.sig_limit;
+        let continuous_perf = self.drift_per_day / weight;
+        (discrete_perf + continuous_perf).sqrt()
+    }
+
+    fn compute_sig_drift(&self, weight: f64, delta_secs: f64) -> f64 {
+        let discrete_drift = weight * self.sig_limit * self.sig_limit;
+        let continuous_drift = self.drift_per_day * delta_secs / SECS_PER_DAY;
+        (discrete_drift + continuous_drift).sqrt()
     }
 
     fn subsample(
@@ -252,10 +264,9 @@ impl RatingSystem for EloMMR {
         let mut base_terms: Vec<(Rating, usize)> = standings
             .par_iter_mut()
             .map(|(player, lo, _)| {
-                let (sig_perf, discrete_drift) =
-                    self.sig_perf_and_drift(params.weight, player.times_played_excl());
-                let continuous_drift = self.drift_per_sec * player.delta_time as f64;
-                let sig_drift = (discrete_drift + continuous_drift).sqrt();
+                let weight = self.compute_weight(params.weight, player.times_played_excl());
+                let sig_perf = self.compute_sig_perf(weight);
+                let sig_drift = self.compute_sig_drift(weight, player.delta_time as f64);
                 match self.variant {
                     // if transfer_speed is infinite or the prior is Gaussian, the logistic
                     // weights become zero so this special-case optimization clears them out
@@ -318,7 +329,8 @@ impl RatingSystem for EloMMR {
                 );
             }
             let bounds = (-6000.0, 9000.0);
-            let (sig_perf, _) = self.sig_perf_and_drift(params.weight, player.times_played_excl());
+            let weight = self.compute_weight(params.weight, player.times_played_excl());
+            let sig_perf = self.compute_sig_perf(weight);
 
             match self.variant {
                 EloMMRVariant::Gaussian => {
diff --git a/multi-skill/src/systems/mod.rs b/multi-skill/src/systems/mod.rs
@@ -21,6 +21,8 @@ pub use simple_elo_mmr::SimpleEloMMR;
 pub use topcoder_sys::TopcoderSys;
 pub use true_skill::TrueSkillSPb;
 
+pub static SECS_PER_DAY: f64 = 86_400.;
+
 // TODO: add a version that can take parameters, like in experiment_config but polymorphic
 pub fn get_rating_system_by_name(
     system_name: &str,
diff --git a/multi-skill/src/systems/simple_elo_mmr.rs b/multi-skill/src/systems/simple_elo_mmr.rs
@@ -1,6 +1,6 @@
 //! This version has fewer features and optimizations than elo_mmr.rs, more
 //! closely matching the pseudocode in https://arxiv.org/abs/2101.00400
-use super::{Player, Rating, RatingSystem, TanhTerm};
+use super::{Player, Rating, RatingSystem, TanhTerm, SECS_PER_DAY};
 use crate::data_processing::ContestRatingParams;
 use crate::numerical::solve_newton;
 use rayon::prelude::*;
@@ -29,8 +29,8 @@ pub struct SimpleEloMMR {
     // each contest participation adds an amount of drift such that, in the absence of
     // much time passing, the limiting skill uncertainty's square approaches this value
     pub sig_limit: f64,
-    // additional variance per second, from a drift that's continuous in time
-    pub drift_per_sec: f64,
+    // additional variance per day, from a drift that's continuous in time
+    pub drift_per_day: f64,
     // whether to count ties as half a win plus half a loss
     pub split_ties: bool,
     // maximum number of recent contests to store, must be at least 1
@@ -43,9 +43,9 @@ impl Default for SimpleEloMMR {
     fn default() -> Self {
         Self {
             weight_limit: 0.2,
-            noob_delay: vec![0.6, 0.8], // TODO: make the default empty once it's configurable
+            noob_delay: vec![],
             sig_limit: 80.,
-            drift_per_sec: 0.,
+            drift_per_day: 0.,
             split_ties: false,
             history_len: usize::MAX,
             transfer_speed: 1.,
@@ -54,21 +54,32 @@ impl Default for SimpleEloMMR {
 }
 
 impl SimpleEloMMR {
-    fn sig_perf_and_drift(&self, mut contest_weight: f64, n: usize) -> (f64, f64) {
+    fn compute_weight(&self, mut contest_weight: f64, n: usize) -> f64 {
         contest_weight *= self.weight_limit;
-        contest_weight *= self.noob_delay.get(n).unwrap_or(&1.);
-        let sig_perf = (1. + 1. / contest_weight).sqrt() * self.sig_limit;
-        let sig_drift_sq = contest_weight * self.sig_limit * self.sig_limit;
-        (sig_perf, sig_drift_sq)
+        if let Some(delay_factor) = self.noob_delay.get(n) {
+            contest_weight *= delay_factor;
+        }
+        contest_weight
+    }
+
+    fn compute_sig_perf(&self, weight: f64) -> f64 {
+        let discrete_perf = (1. + 1. / weight) * self.sig_limit * self.sig_limit;
+        let continuous_perf = self.drift_per_day / weight;
+        (discrete_perf + continuous_perf).sqrt()
+    }
+
+    fn compute_sig_drift(&self, weight: f64, delta_secs: f64) -> f64 {
+        let discrete_drift = weight * self.sig_limit * self.sig_limit;
+        let continuous_drift = self.drift_per_day * delta_secs / SECS_PER_DAY;
+        (discrete_drift + continuous_drift).sqrt()
     }
 }
 
 impl RatingSystem for SimpleEloMMR {
     fn individual_update(&self, params: ContestRatingParams, player: &mut Player, mu_perf: f64) {
-        let (sig_perf, discrete_drift) =
-            self.sig_perf_and_drift(params.weight, player.times_played_excl());
-        let continuous_drift = self.drift_per_sec * player.delta_time as f64;
-        let sig_drift = (discrete_drift + continuous_drift).sqrt();
+        let weight = self.compute_weight(params.weight, player.times_played_excl());
+        let sig_perf = self.compute_sig_perf(weight);
+        let sig_drift = self.compute_sig_drift(weight, player.delta_time as f64);
         player.add_noise_best(sig_drift, self.transfer_speed);
 
         player.update_rating_with_logistic(
@@ -92,10 +103,9 @@ impl RatingSystem for SimpleEloMMR {
         let tanh_terms: Vec<TanhTerm> = standings
             .par_iter_mut()
             .map(|(player, _, _)| {
-                let (sig_perf, discrete_drift) =
-                    self.sig_perf_and_drift(params.weight, player.times_played_excl());
-                let continuous_drift = self.drift_per_sec * player.delta_time as f64;
-                let sig_drift = (discrete_drift + continuous_drift).sqrt();
+                let weight = self.compute_weight(params.weight, player.times_played_excl());
+                let sig_perf = self.compute_sig_perf(weight);
+                let sig_drift = self.compute_sig_drift(weight, player.delta_time as f64);
                 player.add_noise_best(sig_drift, self.transfer_speed);
                 player.approx_posterior.with_noise(sig_perf).into()
             })
@@ -116,7 +126,8 @@ impl RatingSystem for SimpleEloMMR {
                     .fold((0., 0.), |(s, sp), (v, vp)| (s + v, sp + vp))
             };
             let mu_perf = solve_newton(bounds, f).min(params.perf_ceiling);
-            let (sig_perf, _) = self.sig_perf_and_drift(params.weight, player.times_played_excl());
+            let weight = self.compute_weight(params.weight, player.times_played_excl());
+            let sig_perf = self.compute_sig_perf(weight);
             player.update_rating_with_logistic(
                 Rating {
                     mu: mu_perf,
