More detailed evaluation

Author: sagerpascal

requirements.txt

@@ -9,4 +9,5 @@ scipy~=1.10.1
 sphinx~=7.0.1
 sphinx-press-theme~=0.8.0
 sphinxcontrib-video~=0.1.1
-deepdiff~=6.3.0
+deepdiff~=6.3.0
+opencv-python~=4.8.1.78

src/main_evaluation.py

@@ -34,7 +34,7 @@ def parse_args(parser: Optional[argparse.ArgumentParser] = None) -> argparse.Arg
     parser.add_argument("--n_samples",
                         type=int,
                         metavar="N",
-                        default=180,
+                        default=60,
                         help="Number of samples to evaluate."
                         )
     parser.add_argument('--simplified',
@@ -358,9 +358,17 @@ def analyze_interrupt_line(img: Tensor, baseline_img: Tensor, lateral_features:
     """
     baseline_img = baseline_img[-1].squeeze()
     mask = torch.where(img != baseline_img, True, False).unsqueeze(0).repeat(lateral_features[-1].shape[1], 1, 1)
-    baseline_lateral_features = baseline_lateral_features[-1].squeeze(0)[mask]
-    lateral_features = lateral_features[-1].squeeze(0)[mask]
-    accuracy = 1. - F.l1_loss(lateral_features, baseline_lateral_features)
+    baseline_lateral_features_masked = baseline_lateral_features[-1].squeeze(0)[mask]
+    lateral_features_masked = lateral_features[-1].squeeze(0)[mask]
+    mask_active = torch.where(baseline_lateral_features_masked > 0., True, False)
+    if baseline_lateral_features_masked[mask_active].numel() > 0:
+        accuracy = 1. - F.l1_loss(lateral_features_masked[mask_active], baseline_lateral_features_masked[mask_active])
+    else:
+        accuracy = torch.tensor(0.)
+    # mask = torch.where(img != baseline_img, True, False)
+    # baseline_lateral_features_masked = torch.clip(torch.sum(baseline_lateral_features[-1].squeeze(0), dim=0)[mask], max=1)
+    # lateral_features_masked = torch.clip(torch.sum(lateral_features[-1].squeeze(0), dim=0)[mask], max=1)
+    # accuracy = 1. - F.l1_loss(lateral_features_masked, baseline_lateral_features_masked)
     return accuracy.item()
 
 
@@ -439,7 +447,6 @@ def process_data(
     """
     Processes the data and store the network activations as video
     :param generator: Data generator
-    :param eval_args: Evaluation arguments
     :param config: Configuration
     :param fabric: Fabric instance
     :param feature_extractor: Feature extractor
@@ -450,7 +457,7 @@ def process_data(
     avg_noise_meter = AverageMeter()
     avg_line_recon_accuracy_meter = AverageMeter()
     avg_recon_accuracy_meter, avg_recon_recall_meter, avg_recon_precision_meter = AverageMeter(), AverageMeter(), AverageMeter()
-    fp = f"../tmp/v2/{config['run']['load_state_path'].split('.')[0]}_{'noise:'+str(config['noise']) if config['noise'] > 0 else 'no-noise'}_li-{config['line_interrupt']}.mp4"
+    fp = f"../tmp/v3/{config['config']}_{'noise:'+str(config['noise']) if config['noise'] > 0 else 'no-noise'}_li-{config['line_interrupt']}.mp4"
     if Path(fp).exists():
         Path(fp).unlink()
     out = cv2.VideoWriter(fp, cv2.VideoWriter_fourcc(*'mp4v'), config['fps'],
@@ -470,9 +477,39 @@ def process_data(
             result = ci.create_image(inp[timestep], inp_features[timestep, 0], l1_act[timestep, 0], l1_act_prob[timestep, 0])
             out.write(cv2.cvtColor(result, cv2.COLOR_RGB2BGR))
     out.release()
+    l1_acts = torch.stack(l1_acts)
     if 'store_baseline_activations_path' in config and config['store_baseline_activations_path'] is not None:
-        torch.save([torch.stack(l1_acts), torch.stack(imgs_)], config['store_baseline_activations_path'])
-    print("Video stored at", fp)
+        torch.save([l1_acts, torch.stack(imgs_)], config['store_baseline_activations_path'])
+    # print("Video stored at", fp)
+
+    calc_overlap = True
+    if calc_overlap:
+        # consider final timestep
+        l1_acts_f = l1_acts[:, -1].squeeze()
+
+        # calculate active / inactive cells
+        avg_activate_cells = torch.mean(torch.sum(l1_acts_f > 0, dim=(1, 2, 3)).float())
+        avg_inactivate_cells = torch.mean(torch.sum(l1_acts_f == 0, dim=(1, 2, 3)).float())
+        print(f"Average activated cells in net fragments: {avg_activate_cells}")
+        print(f"Average inactivated cells in net fragments: {avg_inactivate_cells}")
+
+        # calculate overlap
+        l1_acts_f = l1_acts_f.view(l1_acts_f.shape[0], -1)
+        overlaps, boverlaps = [], []
+        for v in range(l1_acts_f.shape[0]):
+            biggest_overlap = 0
+            for v2 in range(l1_acts_f.shape[0]):
+                if v != v2:
+                    overlap = torch.sum(l1_acts_f[v] * l1_acts_f[v2]) / torch.sum(l1_acts_f[v] + l1_acts_f[v2])
+                    overlaps.append(overlap)
+                    if overlap > biggest_overlap:
+                        biggest_overlap = overlap
+            boverlaps.append(biggest_overlap)
+
+        overlaps = torch.tensor(overlaps)
+        print(f"Average overlap: {torch.mean(overlaps)}")
+        print(f"Average biggest overlap: {torch.mean(torch.tensor(boverlaps))}")
+
     print(f"Average Noise Reduction: {avg_noise_meter.mean}")
     print(f"Average Interrupt Line Reconstruction Accuracy: {avg_line_recon_accuracy_meter.mean}")
     print(f"Average Reconstruction Accuracy: {avg_recon_accuracy_meter.mean}")

src/main_lateral_connections.py

@@ -3,9 +3,7 @@
 from typing import Any, Dict, Optional
 
 import lightning.pytorch as pl
-import numpy as np
 import torch
-import torch.nn.functional as F
 import wandb
 from lightning import Fabric
 from torch import Tensor
@@ -16,8 +14,8 @@
 
 from data import loaders_from_config
 from lateral_connections.feature_extractor.straight_line_pl_modules import FixedFilterFeatureExtractor
-from lateral_connections.s2_rbm import L2RBM
 from lateral_connections.s1_lateral_connections import LateralNetwork
+from lateral_connections.s2_rbm import L2RBM
 from tools import loggers_from_conf
 from tools.store_load_run import load_run, save_run
 from utils import get_config, print_start, print_warn
@@ -216,7 +214,7 @@ def cycle(
             z_float, z = lateral_network(x_in)
 
             # z2, z2_feedback, h, loss = l2.eval_step(z)
-#
+            #
             # if epoch > 10:
             #     mask_active = (z > 0) | (z2_feedback > 0)
             #     if F.mse_loss(z[mask_active], z2_feedback[mask_active]) < .1:
@@ -263,7 +261,7 @@ def cycle(
 
     if store_tensors:
         return features, torch.stack(input_features, dim=1), torch.stack(lateral_features, dim=1), torch.stack(
-            lateral_features_f, dim=1),None, None
+            lateral_features_f, dim=1), None, None
 
 
 def single_train_epoch(
@@ -349,10 +347,10 @@ def single_eval_epoch(
 
     assert not wandb_b or wandb_b and store_plots, "Wandb logging requires storing the plots."
 
-    if False:
+    if plot:
         if epoch == 0:
             feature_extractor.plot_model_weights(show_plot=plot)
-#
+        #
         plots_fp = lateral_network.plot_samples(plt_img,
                                                 plt_features,
                                                 plt_input_features,

src/plot_robustness_paper.py

@@ -4,7 +4,8 @@
 import pandas as pd
 from matplotlib import pyplot as plt
 
-JSON_FP = Path(".").absolute().parent / "tmp" / "experiment_results.json"
+JSON_FP = Path(".").absolute().parent / "tmp" / "experiment_results_li.json"
+# JSON_FP = Path(".").absolute().parent / "tmp" / "experiment_results.json"
 
 
 def replace_square_list(sl):
@@ -27,10 +28,11 @@ def get_data():
             result = {
                 'noise': data['config']['noise'],
                 'line_interrupt': data['config']['line_interrupt'],
-                'act_threshold': data['config']['lateral_model']['l1_params']['act_threshold'],
+                'act_bias': round(data['config']['lateral_model']['l1_params']['act_threshold']-0.5, 2),
                 'square_factor': replace_square_list(data['config']['lateral_model']['l1_params']['square_factor']),
                 'noise_reduction': data['noise_reduction'],
                 'avg_line_recon_accuracy_meter': data['avg_line_recon_accuracy_meter'],
+                'avg_line_recon_accuracy_meter_2': (data['avg_line_recon_accuracy_meter']-0.75)/0.25,
                 'recon_accuracy': data['recon_accuracy'],
                 'recon_recall': data['recon_recall'],
                 'recon_precision': data['recon_precision'],
@@ -46,24 +48,28 @@ def feature_noise_to_location_noise(feature_noise, round_=False):
         result = np.round(result, 2)
     return result
 
-def plot_line(data, x_key, x_label, y_key, y_label, z_key, z_label, plot_key, plot_label, x2_func=None, x2_label=None):
+def plot_line(data, x_key, x_label, y_key, y_label, z_key, z_label, plot_key, plot_label, ymin=None,
+              x2_func=None, x2_label=None):
 
-    fig, axs = plt.subplots(ncols=len(data[plot_key].unique()), figsize=(18, 6), dpi=100)
+    fig, axs = plt.subplots(ncols=len(data[plot_key].unique()), figsize=(13, 3), dpi=100)
 
-    for ax, pk in zip(axs, data[plot_key].unique()):
+    for ax, pk in zip(axs, sorted(data[plot_key].unique())):
         data_ = data[data[plot_key] == pk]
 
         z_values = list(data_[z_key].unique())
         z_values = sorted(z_values)
 
         for zv in z_values:
             z = data_[data_[z_key] == zv]
-            ax.plot(z[x_key].values, z[y_key].values, label="{} = {}".format(z_label, zv))
+            ax.plot(z[x_key].values, z[y_key].values, label="{}{}".format(z_label, zv))
 
-        ax.set_title("{} = {}".format(plot_label, pk))
+        ax.set_title("{}{}".format(plot_label, pk))
         ax.set_xlabel(x_label)
         ax.set_ylabel(y_label)
 
+        if ymin is not None:
+            ax.set_ylim(ymax=1.0+0.02, ymin=ymin-0.02)
+
         if x2_func is not None:
             ax2 = ax.twiny()
             ax2.set_xlim(ax.get_xlim())
@@ -74,6 +80,7 @@ def plot_line(data, x_key, x_label, y_key, y_label, z_key, z_label, plot_key, pl
         ax.legend()
         ax.grid()
 
+    plt.tight_layout()
     plt.show()
 
 
@@ -85,26 +92,31 @@ def plot(data):
     # plot noise only
     data_1 = data[data['line_interrupt'] == 0]
     plot_line(data_1, x_key="noise", x_label="Feature Noise", y_key="noise_reduction", y_label="Noise Reduction Rate",
-              z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor',
-              x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
+              z_key='act_bias', z_label='Act. Bias b = ', plot_key='square_factor', plot_label='Power Factor γ = ',
+              ymin=0.4, x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
 
     plot_line(data_1, x_key="noise", x_label="Feature Noise", y_key="recon_recall", y_label="Recall",
-              z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor',
-              x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
+              z_key='act_bias', z_label='Act. Bias b = ', plot_key='square_factor', plot_label='Power Factor γ = ',
+              ymin=0.2, x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
 
     plot_line(data_1, x_key="noise", x_label="Feature Noise", y_key="recon_precision", y_label="Precision",
-               z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor',
-              x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
+               z_key='act_bias', z_label='Act. Bias b = ', plot_key='square_factor', plot_label='Power Factor γ = ',
+              ymin=0.1, x2_func=feature_noise_to_location_noise, x2_label="Spatial Noise")
 
     # plot line interrupt only
     data_1 = data[data['noise'] == 0.0]
     plot_line(data_1, x_key="line_interrupt", x_label="Line Interrupt", y_key="avg_line_recon_accuracy_meter",
-              y_label="Feature Reconstruction Rate",
-              z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor')
+              y_label="Feature Reconstruction Rate", z_key='act_bias', z_label='Act. Bias b = ',
+              plot_key='square_factor', plot_label='Power Factor γ = ', ymin=0.0)
+    # plot_line(data_1, x_key="line_interrupt", x_label="Line Interrupt", y_key="avg_line_recon_accuracy_meter_2",
+    #           y_label="Feature Reconstruction Rate 2", z_key='act_bias', z_label='Act. Bias b = ',
+    #           plot_key='square_factor', plot_label='Power Factor γ = ', ymin=0.0)
     plot_line(data_1, x_key="line_interrupt", x_label="Line Interrupt", y_key="recon_recall", y_label="Recall",
-              z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor')
+              z_key='act_bias', z_label='Act. Bias b = ', plot_key='square_factor', plot_label='Power Factor γ = ',
+              ymin=0.5)
     plot_line(data_1, x_key="line_interrupt", x_label="Line Interrupt", y_key="recon_precision", y_label="Precision",
-              z_key='act_threshold', z_label='Act. Threshold', plot_key='square_factor', plot_label='Square Factor')
+              z_key='act_bias', z_label='Act. Bias b = ', plot_key='square_factor', plot_label='Power Factor γ = ',
+              ymin=0.6)
 
 
 if __name__ == '__main__':