almost ready for offline training

Author: munikarmanish

.gitignore

@@ -119,5 +119,6 @@ ENV/
 
 # End of https://www.gitignore.io/api/linux,python
 
-trees
+#trees
 models
+log.csv

main.py

@@ -40,10 +40,10 @@ def main():
         print("Testing...")
         model = rntn.RNTN.load(args.model)
         test_trees = tr.load_trees(args.dataset)
-        cost, correct, total = model.test(test_trees)
-        accuracy = correct * 100.0 / total
+        cost, result = model.test(test_trees)
+        accuracy = 100.0 * result.trace() / result.sum()
         print("Cost = {:.2f}, Correct = {:.0f} / {:.0f}, Accuracy = {:.2f} %".format(
-            cost, correct, total, accuracy))
+            cost, result.trace(), result.sum(), accuracy))
     else:
         # Initialize the model
         model = rntn.RNTN(

rntn.py

@@ -38,8 +38,9 @@ def fit(self, trees, export_filename='models/RNTN.pickle', verbose=False):
         with open("log.csv", "a", newline='') as csvfile:
             csvwriter = csv.writer(csvfile)
             fieldnames = ["Timestamp", "Vector size", "Learning rate",
-                          "Batch size", "Regularization", "Epoch", "Cost",
-                          "Accuracy"]
+                          "Batch size", "Regularization", "Epoch",
+                          "Train cost", "Train accuracy",
+                          "Test cost", "Test accuracy"]
             if csvfile.tell() == 0:
                 csvwriter.writerow(fieldnames)
 
@@ -53,13 +54,17 @@ def fit(self, trees, export_filename='models/RNTN.pickle', verbose=False):
                 # Save the model
                 self.save(export_filename)
 
-                # Test the model
-                cost, correct, total = self.test(test_trees)
-                accuracy = correct * 100.0 / total
+                # Test the model on train and test set
+                train_cost, train_result = self.test(trees)
+                train_accuracy = 100.0 * train_result.trace() / train_result.sum()
+                test_cost, test_result = self.test(test_trees)
+                test_accuracy = 100.0 * test_result.trace() / test_result.sum()
 
                 # Append data to CSV file
                 row = [datetime.now(), self.dim, self.learning_rate,
-                       self.batch_size, self.reg, epoch, cost, accuracy]
+                       self.batch_size, self.reg, epoch,
+                       train_cost, train_accuracy,
+                       test_cost, test_accuracy]
                 csvwriter.writerow(row)
 
     def test(self, trees):
@@ -145,18 +150,17 @@ def init_params(self):
         self.dbs = np.empty_like(self.bs)
 
     def cost_and_grad(self, trees, test=False):
-        cost, correct, total = 0.0, 0.0, 0.0
+        cost, result = 0.0, np.zeros((5,5))
         self.L, self.V, self.W, self.b, self.Ws, self.bs = self.stack
 
         # Forward propagation
         for tree in trees:
-            _cost, _correct, _total = self.forward_prop(tree)
+            _cost, _result = self.forward_prop(tree)
             cost += _cost
-            correct += _correct
-            total += _total
+            result += _result
 
         if test:
-            return cost / len(trees), correct, total
+            return cost / len(trees), result
 
         # Initialize gradients
         self.dL = collections.defaultdict(lambda: np.zeros((self.dim,)))
@@ -191,7 +195,8 @@ def cost_and_grad(self, trees, test=False):
         return cost, grad
 
     def forward_prop(self, tree):
-        cost, correct, total = 0.0, 0.0, 0.0
+        cost = 0.0
+        result = np.zeros((5,5))
 
         if tr.isleaf(tree):
             # output = word vector
@@ -202,11 +207,10 @@ def forward_prop(self, tree):
             tree.fprop = True
         else:
             # calculate output of child nodes
-            lcost, lcorrect, ltotal = self.forward_prop(tree[0])
-            rcost, rcorrect, rtotal = self.forward_prop(tree[1])
+            lcost, lresult = self.forward_prop(tree[0])
+            rcost, rresult = self.forward_prop(tree[1])
             cost += lcost + rcost
-            correct += lcorrect + rcorrect
-            total += ltotal + rtotal
+            result += lresult + rresult
 
             # compute output
             lr = np.hstack([tree[0].vector, tree[1].vector])
@@ -224,10 +228,11 @@ def forward_prop(self, tree):
 
         # cost
         cost -= np.log(tree.output[int(tree.label())])
-        correct += (np.argmax(tree.output) == int(tree.label()))
-        total += 1
+        true_label = int(tree.label())
+        predicted_label = np.argmax(tree.output)
+        result[true_label, predicted_label] += 1
 
-        return cost, correct, total
+        return cost, result
 
     def back_prop(self, tree, error=None):
         # clear nodes
@@ -245,7 +250,11 @@ def back_prop(self, tree, error=None):
 
         # leaf node => update word vectors
         if tr.isleaf(tree):
-            self.dL[self.word_map[tree[0]]] += deltas
+            try:
+                index = self.word_map[tree[0]]
+            except KeyError:
+                index = self.word_map[tr.UNK]
+            self.dL[index] += deltas
             return
 
         # Hidden gradients

test.py

@@ -1,7 +1,5 @@
 #!/bin/env python3
 
-import unittest
-
 import tree as tr
 
 
@@ -21,17 +19,13 @@ def f(model, text):
     >>> model = rntn.RNTN.load('models/RNTN.pickle')
     >>> f(model, "not very good")
            1
-      -----|-----
+       ____|____
       |         4
-      |      ---|---
+      |       __|__
       2      2     3
       |      |     |
      not   very   good
 
     """
     for tree in tr.parse(text):
         model.predict(tree).pretty_print()
-
-
-if __name__ == '__main__':
-    unittest.main()

train.sh

@@ -1,22 +1,79 @@
 #!/bin/sh
 
-set -x
-
-dim=25
-epochs=30
-learning_rate=1e-2
-batch_size=30
-optimizer="adagrad"
-reg=1e-6
+# The default values
+DEFAULT_DIM=25
+DEFAULT_EPOCH=10
+DEFAULT_LEARNING_RATE=1e-1
+DEFAULT_BATCH_SIZE=30
+DEFAULT_REG=1e-6
+
+# Values to test
+DIM_LIST=( 10 20 25 30 40 50 )
+LEARNING_RATE_LIST=( 1e-4 1e-3 1e-2 1e-1 1 )
+BATCH_SIZE_LIST=( 1 10 30 50 70 100 )
+REG_LIST=( 1e-6 1e-4 1e-2 0 10 )
+
+optimizer="adagrad"     # This is like constant
+
+# Tune vector size
+#------------------
+
+epochs=$DEFAULT_EPOCH
+learning_rate=$DEFAULT_LEARNING_RATE
+batch_size=$DEFAULT_BATCH_SIZE
+reg=$DEFAULT_REG
 datetime=$(date +"%Y%m%d%H%M")
-dataset="train"
+for dim in "${DIM_LIST[@]}"; do
+    outfile="models/RNTN_D${dim}_E${epochs}_B${batch_size}_L${learning_rate}_R${reg}_${optimizer}_${datetime}.pickle"
+    set -x
+    python3 main.py \
+        --dim=${dim} \
+        --epochs=${epochs} \
+        --learning-rate=${learning_rate} \
+        --batch-size=${batch_size} \
+        --reg=${reg} \
+        --model=${outfile}
+    set +x
+done
 
-outfile="models/RNTN_D${dim}_E${epochs}_B${batch_size}_L${learning_rate}_R${reg}_${optimizer}_${datetime}.pickle"
+# Tune batch size
+#---------------------
 
-python3 main.py \
-    --dim=${dim} \
-    --epochs=${epochs} \
-    --learning-rate=${learning_rate} \
-    --batch-size=${batch_size} \
-    --dataset=${dataset} \
-    --model=${outfile}
+epochs=$DEFAULT_EPOCH
+learning_rate=$DEFAULT_LEARNING_RATE
+dim=$DEFAULT_DIM
+reg=$DEFAULT_REG
+datetime=$(date +"%Y%m%d%H%M")
+for batch_size in "${BATCH_SIZE_LIST[@]}"; do
+    outfile="models/RNTN_D${dim}_E${epochs}_B${batch_size}_L${learning_rate}_R${reg}_${optimizer}_${datetime}.pickle"
+    set -x
+    python3 main.py \
+        --dim=${dim} \
+        --epochs=${epochs} \
+        --learning-rate=${learning_rate} \
+        --batch-size=${batch_size} \
+        --reg=${reg} \
+        --model=${outfile}
+    set +x
+done
+
+# Tune regularization parameter
+#-------------------------------
+
+epochs=$DEFAULT_EPOCH
+learning_rate=$DEFAULT_LEARNING_RATE
+dim=$DEFAULT_DIM
+batch_size=$DEFAULT_BATCH_SIZE
+datetime=$(date +"%Y%m%d%H%M")
+for reg in "${REG_LIST[@]}"; do
+    outfile="models/RNTN_D${dim}_E${epochs}_B${batch_size}_L${learning_rate}_R${reg}_${optimizer}_${datetime}.pickle"
+    set -x
+    python3 main.py \
+        --dim=${dim} \
+        --epochs=${epochs} \
+        --learning-rate=${learning_rate} \
+        --batch-size=${batch_size} \
+        --reg=${reg} \
+        --model=${outfile}
+    set +x
+done