update with PyKEEN 1.8.1-dev0 (6ac40cd5)

add geometric mean rank adjustments
pykeen · Mar 29, 2022 · 18427ce · 18427ce
1 parent d50af9c
commit 18427ce
Show file tree

Hide file tree

Showing 15 changed files with 3,951 additions and 3,191 deletions.
diff --git a/charts/arithmetic_mean_rank_plot.pdf b/charts/arithmetic_mean_rank_plot.pdf
diff --git a/charts/arithmetic_mean_rank_plot.png b/charts/arithmetic_mean_rank_plot.png
diff --git a/charts/arithmetic_mean_rank_plot.svg b/charts/arithmetic_mean_rank_plot.svg
diff --git a/charts/hits_at_10_plot.pdf b/charts/hits_at_10_plot.pdf
diff --git a/charts/hits_at_10_plot.png b/charts/hits_at_10_plot.png
diff --git a/charts/hits_at_10_plot.svg b/charts/hits_at_10_plot.svg
diff --git a/charts/mean_reciprocal_rank_plot.pdf b/charts/mean_reciprocal_rank_plot.pdf
diff --git a/charts/mean_reciprocal_rank_plot.png b/charts/mean_reciprocal_rank_plot.png
diff --git a/charts/mean_reciprocal_rank_plot.svg b/charts/mean_reciprocal_rank_plot.svg
diff --git a/charts/summary.png b/charts/summary.png
diff --git a/charts/summary.svg b/charts/summary.svg
diff --git a/collated/collated.tsv b/collated/collated.tsv
diff --git a/collated/melted.tsv b/collated/melted.tsv
diff --git a/scripts/fix.py b/scripts/fix.py
@@ -15,6 +15,7 @@
         continue
     original_column = column[2:]
     pairs[column] = original_column
+pairs["z_geometric_mean_rank"] = "geometric_mean_rank"
 
 hits_prefix = "hits_at_"
 expectation, variance = {}, {}
@@ -35,14 +36,23 @@
         kwargs = {}
         norm_metric_name = metric_name
         if hits_prefix in metric_name:
-            kwargs["k"] = int(metric_name[len(hits_prefix):])
+            kwargs["k"] = int(metric_name[len(hits_prefix) :])
             norm_metric_name = hits_prefix
         metric = rank_based_metric_resolver.make(norm_metric_name, pos_kwargs=kwargs)
         mean = metric.expected_value(num_candidates=num_candidates)
         std = metric.std(num_candidates=num_candidates)
-        df.loc[df["dataset"] == dataset_name, z_metric_name] = (
-                                                                   df.loc[df["dataset"] == dataset_name, metric_name] - mean
-                                                               ) / std
+        value = (df.loc[df["dataset"] == dataset_name, metric_name] - mean) / std
+        if "geometric_mean_rank" in metric_name:
+            value *= -1
+        df.loc[df["dataset"] == dataset_name, z_metric_name] = value
+    # adjusted geometric mean rank index
+    metric_name = "geometric_mean_rank"
+    metric = rank_based_metric_resolver.make(metric_name)
+    mean = metric.expected_value(num_candidates=num_candidates)
+    std = metric.std(num_candidates=num_candidates)
+    value = (df.loc[df["dataset"] == dataset_name, metric_name] - mean) / (1 - mean)
+    df.loc[df["dataset"] == dataset_name, "adjusted_geometric_mean_rank_index"] = value
+
 df.to_csv("collated/collated.tsv", sep="\t", index=False)
 
 id_vars = ["dataset", "dataset_triples", "model"]

diff --git a/scripts/plot.py b/scripts/plot.py
@@ -73,6 +73,17 @@ def _lookup_key(d):
         ],
         "short": ["$H_{10}$", "$AH_{10}$", "$ZH_{10}$"],
     },
+    "geometric_mean_rank": {
+        "base_title": "Geometric Mean Rank",
+        "base_yscale": "log",
+        "metrics": [
+            "geometric_mean_rank",
+            "adjusted_geometric_mean_rank_index",
+            "z_geometric_mean_rank",
+        ],
+        "short": ["GMR", "AGMRI", "ZGMR"],
+        "has_negative_z": True,
+    }
 }
 
 
@@ -115,7 +126,7 @@ def main():
                 else:
                     ax.set_yscale("log")
             elif key.startswith("Adjusted Index"):
-                ax.set_ylim([0, 1])
+                ax.set_ylim([-0.1, 1])
             else:  # base metric
                 if "base_ylim" in metadata:
                     ax.set_ylim(metadata["base_ylim"])