wip

Author: acharneski

docs/result_analysis.md

@@ -15,11 +15,16 @@
 \usepackage{multirow}
 \usepackage{adjustbox}
 \usepackage{siunitx}
+\usepackage{makecell}
+\providecommand{\tightlist}{%
+    \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
 
 % Fix any Unicode issues
 \DeclareUnicodeCharacter{2212}{-}
 \DeclareUnicodeCharacter{2264}{\leq}
 \DeclareUnicodeCharacter{2265}{\geq}
+\definecolor{bestgreen}{RGB}{0,150,0}
+\definecolor{worstred}{RGB}{200,0,0}
 
 \title{QQN: A Quadratic Hybridization of Quasi-Newton Methods for Nonlinear Optimization}
 \author{Andrew Charneski\\SimiaCryptus Software}

paper/combined.md

@@ -340,7 +340,7 @@ impl PlottingEngine {
         }
 
         // Color palette for different optimizers
-        let colors = [
+        let colors: [&RGBColor; 24] = [
             &RED,
             &BLUE,
             &GREEN,
@@ -372,13 +372,13 @@ impl PlottingEngine {
         for (optimizer_idx, optimizer_name) in unique_optimizers.iter().enumerate() {
             let color = colors[optimizer_idx % colors.len()];
             let traces_for_optimizer = &optimizer_traces[optimizer_name];
-            debug!(
-                "Drawing {} traces for optimizer: {}",
+            info!(
+                "Drawing {} traces/runs for optimizer: {}",
                 traces_for_optimizer.len(),
                 optimizer_name
             );
 
-            for trace in traces_for_optimizer {
+            for (run_idx, trace) in traces_for_optimizer.iter().enumerate() {
                 let series_data: Vec<(usize, f64)> = trace
                     .evaluation_counts
                     .iter()
@@ -390,15 +390,23 @@ impl PlottingEngine {
                     debug!("Skipping empty series for {}", optimizer_name);
                     continue;
                 }
-                debug!(
-                    "Drawing series with {} data points for {}",
+                info!(
+                    "Drawing series with {} data points for {} (run {})",
                     series_data.len(),
-                    optimizer_name
+                    optimizer_name,
+                    run_idx
                 );
+                // Use slightly different line styles for multiple runs of the same optimizer
+                let line_style = if run_idx == 0 {
+                   (*color).to_rgba()
+                } else {
+                    // Make subsequent runs more transparent based on run index
+                   (*color).mix(0.8 - (run_idx as f64 * 0.05).min(0.3)).to_rgba()
+                };
 
                 // Draw series
                 chart
-                    .draw_series(LineSeries::new(series_data.clone(), color))
+                    .draw_series(LineSeries::new(series_data.clone(), &line_style))
                     .map_err(|e| anyhow::anyhow!("Series drawing error: {}", e))?;
                 // Add markers at regular intervals for better visibility
                 let marker_interval = series_data.len().max(1) / 20 + 1;
@@ -408,7 +416,7 @@ impl PlottingEngine {
                         series_data
                             .iter()
                             .step_by(marker_interval)
-                            .map(|&(x, y)| Circle::new((x, y), 3, color.filled())),
+                            .map(|&(x, y)| Circle::new((x, y), 3, line_style.filled())),
                     )
                     .map_err(|e| anyhow::anyhow!("Marker drawing error: {}", e))?;
             }
@@ -606,7 +614,7 @@ impl PlottingEngine {
                 optimizer_name
             );
 
-            for trace in traces_for_optimizer {
+            for (run_idx, trace) in traces_for_optimizer.iter().enumerate() {
                 let series_data: Vec<(usize, f64)> = trace
                     .evaluation_counts
                     .iter()
@@ -622,11 +630,18 @@ impl PlottingEngine {
                 // Only draw if we have valid data points
                 if !series_data.is_empty() {
                     debug!(
-                        "Drawing log series with {} data points for {}",
+                        "Drawing log series with {} data points for {} (run {})",
                         series_data.len(),
-                        optimizer_name
+                        optimizer_name,
+                        run_idx
                     );
-                    chart.draw_series(LineSeries::new(series_data.clone(), color))?;
+                    // Use slightly different line styles for multiple runs of the same optimizer
+                    let line_style = if run_idx == 0 {
+                       (*color).to_rgba()
+                    } else {
+                        // Make subsequent runs slightly more transparent
+                       (*color).mix(0.7).to_rgba()
+                    };
 
                     // Add markers for better visibility
                     let marker_interval = series_data.len().max(1) / 20 + 1;
@@ -635,7 +650,7 @@ impl PlottingEngine {
                         series_data
                             .iter()
                             .step_by(marker_interval)
-                            .map(|&(x, y)| Circle::new((x, y), 3, color.filled())),
+                            .map(|&(x, y)| Circle::new((x, y), 3, line_style.filled())),
                     )?;
                 } else {
                     debug!("Skipping empty log series for {}", optimizer_name);
@@ -1487,4 +1502,4 @@ mod tests {
         assert!(data.q1 < data.median);
         assert!(data.median < data.q3);
     }
-}
+}

paper/content.tex

@@ -331,6 +331,12 @@ impl BenchmarkRunner {
         mut optimizers: Vec<Box<dyn Optimizer>>,
     ) -> Result<BenchmarkResults, BenchmarkError> {
         let mut results = BenchmarkResults::new(self.config.clone());
+        info!(
+            "Running benchmarks with {} problems, {} optimizers, {} runs each",
+            problems.len(),
+            optimizers.len(),
+            self.config.num_runs
+        );
 
         for problem in &problems {
             for optimizer in &mut optimizers {
@@ -349,6 +355,10 @@ impl BenchmarkRunner {
                 }
             }
         }
+        info!(
+            "Benchmark complete: collected {} total results",
+            results.results.len()
+        );
 
         Ok(results)
     }
@@ -1086,4 +1096,4 @@ impl ProblemSpec {
             .clone()
             .unwrap_or_else(|| self.problem.name().to_string())
     }
-}
+}

paper/paper.pdf

@@ -40,6 +40,11 @@ impl PlottingManager {
         // Generate convergence plots for each problem
         for (problem, results) in all_results {
             let problem_name = problem.get_name();
+            info!(
+                "Processing problem '{}' with {} total results",
+                problem_name,
+                results.results.len()
+            );
             let traces: Vec<ExtendedOptimizationTrace> = results
                 .results
                 .iter()
@@ -63,7 +68,7 @@ impl PlottingManager {
 
             if !traces.is_empty() {
                 info!(
-                    "Generating plots for {} with {} optimizers",
+                    "Generating plots for {} with {} total traces",
                     problem_name,
                     traces.len()
                 );
@@ -143,4 +148,4 @@ impl PlottingManager {
             }
         }
     }
-}
+}

paper/paper.tex

@@ -38,7 +38,7 @@ async fn calibration() -> Result<(), Box<dyn Error + Send + Sync>> {
 
 #[tokio::test]
 async fn full_test() -> Result<(), Box<dyn Error + Send + Sync>> {
-    // init_logging(false)?;
+    init_logging(false)?;
     // Disable no threshold mode for this test
     disable_no_threshold_mode();
 
@@ -64,9 +64,9 @@ async fn test_all(
     prefix: &str,
     problems: Vec<ProblemSpec>,
 ) -> Result<(), Box<dyn Error + Send + Sync>> {
-    let max_evals = 10000;
-    let num_runs = 10;
-    let max_cpu = Some(8);
+    let max_evals = 100;
+    let num_runs = 5;
+    let max_cpu = Some(1);
     run_benchmark(
         &format!("{prefix}all_optimizers_"),
         max_evals,