some update to make it run

.gitignore

@@ -0,0 +1,33 @@
+*.pyc
+
+*.zip
+
+*.pkl
+
+*.tar.gz
+
+*.npz
+
+*.swp
+
+*.txt
+
+logs*
+
+checkpoint*
+
+# some data
+data-prepro/MSCOCO_preprocess/deep-residual-networks
+data-prepro/MSCOCO_preprocess/neuraltalk2
+data-prepro/MSCOCO_preprocess/resnet_model
+data-prepro/MSCOCO_preprocess/mscoco_data
+data-prepro/MSCOCO_preprocess/cub_data
+data-prepro/MSCOCO_preprocess/annotations
+data-prepro/CUB200_preprocess
+data-prepro/topic/
+
+show-adapt-tell/coco_spice
+
+topic-adapt-tell/coco_spice
+topic-adapt-tell/results*
+topic-adapt-tell/log*

.gitmodules

@@ -4,6 +4,3 @@
 [submodule "data-prepro/MSCOCO_preprocess/deep-residual-networks"]
 	path = data-prepro/MSCOCO_preprocess/deep-residual-networks
 	url = [email protected]:KaimingHe/deep-residual-networks.git
-[submodule "show-adapt-tell/coco-caption"]
-	path = show-adapt-tell/coco-caption
-	url = [email protected]:peteanderson80/coco-caption.git

README.md

@@ -40,15 +40,16 @@ If you find this code useful for your research, please cite
 ## Requirements
 
 - Python 2.7
-- [Tensoflow 0.12.1](https://www.tensorflow.org/versions/r0.12/get_started/os_setup)
+- ~~[Tensoflow 0.12.1](https://www.tensorflow.org/versions/r0.12/get_started/os_setup)~~
+- [Tensorflow 1.4](https://www.tensorflow.org/versions/r1.4/)
 - [Caffe](https://github.com/BVLC/caffe)
 - OpenCV 2.4.9
 
 P.S. Please clone the repository with the `--recursive` flag:
 
   ```Shell
   # Make sure to clone with --recursive
-  git clone --recursive https://github.com/tsenghungchen/show-adapt-and-tell.git
+  git clone --recursive https://github.com/Smellly/show-adapt-and-tell.git
   ```
 
 ## Data Preprocessing

data-prepro/CUB200_preprocess/CUB_preprocess_token.py → data-prepro/CUB200_preprocess/cub_preprocess_token.py

@@ -2,29 +2,34 @@
 import json
 import numpy as np
 from tqdm import tqdm
-import pdb
+# import pdb
 import os
-import pickle
-import cPickle
+try: 
+    import cPickle as pkl
+except:
+    import pickle as pkl
 import string
+import sys
 
 def unpickle(p):
-    return cPickle.load(open(p,'r'))
+    return pkl.load(open(p,'r'))
 
 def load_json(p):
     return json.load(open(p,'r'))
 
 def clean_words(data):
-    dict = {}
+    d = {}
     freq = {}
     # start with 1
     idx = 1
     sentence_count = 0
     eliminate = 0
     max_w = 30
-    for k in tqdm(range(len(data['caption']))):
-        sen = data['caption'][k]
-        filename = data['file_name'][k]
+    # for k in tqdm(range(len(data['caption']))):
+    for k in tqdm(data.keys()):
+        # print k, type(k), data[k]
+        sen = data[k][0]['caption']
+        filename = data[k][0]['filename']
         # skip the no image description
         words = re.split(' ', sen)
         # pop the last u'.'
@@ -36,24 +41,31 @@ def clean_words(data):
                     if p in word:
                         word = word.replace(p,'')
                 word = word.lower()
-                if word not in dict.keys():
-                    dict[word] = idx
+                if word not in d.keys():
+                    d[word] = idx
                     idx += 1
                     freq[word] = 1
                 else:
                     freq[word] += 1
         else:
             eliminate += 1
+
     print 'Threshold(max_words) =', max_w
     print 'Eliminate =', eliminate 
     print 'Total sentence_count =', sentence_count
-    print 'Number of different words =', len(dict.keys())
+    print 'Number of different words =', len(d.keys())
     print 'Saving....'
-    np.savez('cleaned_words', dict=dict, freq=freq)
-    return dict, freq
+
+    np.savez('K_cleaned_words', dict=d, freq=freq)
+
+    return d, freq
 
 
-phase = 'train'
+phase = sys.argv[1]
+data_path = './cub_data/K_' + phase + '_annotation.json'
+data = load_json(data_path)
+# print type(data), data.keys()
+
 id2name = unpickle('id2name.pkl')
 id2caption = unpickle('id2caption.pkl')
 splits = unpickle('splits.pkl')
@@ -70,23 +82,34 @@ def clean_words(data):
         caption_list.append(sen)
 
 # build dictionary
-if not os.path.isfile('cub_data/dictionary_'+str(thres)+'.npz'):
-    pdb.set_trace()
+if not os.path.isfile('./cub_data/dictionary_'+str(thres)+'.npz'):
+    # pdb.set_trace()
     # clean the words through the frequency
-    words = np.load('K_cleaned_words.npz')
-    dict = words['dict'].item(0)
-    freq = words['freq'].item(0)
+    if not os.path.isfile('K_cleaned_words.npz'):
+	d, freq = clean_words(data)
+    else:
+        words = np.load('K_cleaned_words.npz')
+        d = words['d'].item(0)
+        freq = words['freq'].item(0)
+
     idx2word = {}
     word2idx = {}
     idx = 1
-    for k in tqdm(dict.keys()):
+    for k in tqdm(d.keys()):
         if freq[k] >= thres:
             word2idx[k] = idx
             idx2word[str(idx)] = k
             idx += 1
 
-    word2idx[u'<UNK>'] = len(word2idx.keys())+1
+    word2idx[u'<BOS>'] = 0
+    idx2word["0"] = u'<BOS>'
+    word2idx[u'<EOS>'] = len(word2idx.keys())
+    idx2word[str(len(idx2word.keys()))] = u'<EOS>'
+    word2idx[u'<UNK>'] = len(word2idx.keys())
     idx2word[str(len(word2idx.keys()))] = u'<UNK>'
+    word2idx[u'<NOT>'] = len(word2idx.keys())
+    idx2word[str(len(idx2word.keys()))] = u'<NOT>'
+
     print 'Threshold of word fequency =', thres
     print 'Total words in the dictionary =', len(word2idx.keys())
     np.savez('cub_data/dictionary_'+str(thres), word2idx=word2idx, idx2word=idx2word)
@@ -104,6 +127,7 @@ def clean_words(data):
 filename_list_new = []
 img_id_list_new = []
 caption_length = []
+
 for k in tqdm(range(len(caption_list))):
     sen = caption_list[k]
     img_id = img_id_list[k]
@@ -143,8 +167,10 @@ def clean_words(data):
 tokenized_caption_info['filename_list'] = np.asarray(filename_list_new)
 tokenized_caption_info['img_id_list'] = np.asarray(img_id_list_new)
 tokenized_caption_info['raw_caption_list'] = np.asarray(caption_list_new)
+
 print 'Number of sentence =', num_sentence
 print 'eliminate = ', eliminate
+
 with open('./cub_data/tokenized_'+phase+'_caption.pkl', 'w') as outfile:
-    pickle.dump(tokenized_caption_info, outfile)
+    pkl.dump(tokenized_caption_info, outfile)
 

data-prepro/CUB200_preprocess/get_split.py

@@ -3,7 +3,7 @@
 import cPickle
 
 # generate name2id & id2name dictionary
-name_id_path = '../images.txt'
+name_id_path = 'cub_data/CUB_200_2011/images.txt'
 name_id = open(name_id_path).read().splitlines()
 name2id = {}
 id2name = {}
@@ -19,6 +19,7 @@
 # CUB_CVPR16 will be created after unzipping. 
 caption_path = './CUB_CVPR16/text_c10/'
 id2caption = {}
+
 for name in name2id:
     txt_name = '.'.join(name.split('.')[0:-1]) + '.txt'
     txt_path = os.path.join(caption_path, txt_name)

data-prepro/CUB200_preprocess/prepro_cub_annotation.py

@@ -6,21 +6,22 @@
 from random import shuffle, seed
 import pickle as pk
 import pdb
-input_data = 'split.pkl'
+
+input_data = 'splits.pkl'
 with open(input_data) as data_file:
     dataset = pk.load(data_file)
 
 skip_num = 0
 val_data = {}
 test_data = {}
-train_data = []
-
+train_data = {}
 val_dataset = []
 test_dataset = []
 counter = 0
 id2name = pk.load(open('id2name.pkl'))
 data = pk.load(open('id2caption.pkl'))
 
+print("Processing test_data")
 for i in dataset['test_id']:
         caps = []
         # For GT
@@ -40,6 +41,58 @@
             caps.append(tmp)
 
         test_data[i] = caps
+
+print 'dump %d in test_data'%len(test_data)
+json.dump(test_data, open('cub_data/K_test_annotation.json', 'w'))
+# pk.dump(test_data, open('cub_data/K_test_annotation.pkl', 'w'))
+
+print("Processing train_data")
+for i in dataset['train_id']:
+        caps = []
+        # For GT
+        name = id2name[i]
+        count = 0
+        for sen in data[i]:
+            for punc in string.punctuation:
+                if punc in sen:
+                    sen = sen.replace(punc, '')
+
+            tmp = {}
+            tmp['filename'] = name
+            tmp['img_id'] = i
+            tmp['cap_id'] = count
+            tmp['caption'] = sen
+            count += 1
+            caps.append(tmp)
+
+        # print i, type(i)
+        train_data[i] = caps
 print 'number of skip train data: ' + str(skip_num)
 [u'info', u'images', u'licenses', u'type', u'annotations']
-json.dump(test_data, open('cub_data/K_test_annotation.json', 'w'))
+print 'dump %d in train_data'%len(train_data)
+json.dump(test_data, open('cub_data/K_train_annotation.json', 'w'))
+# pk.dump(train_data, open('cub_data/K_train_annotation.pkl', 'w'))
+
+print("Processing val_data")
+for i in dataset['val_id']:
+        caps = []
+        # For GT
+        name = id2name[i]
+        count = 0
+        for sen in data[i]:
+            for punc in string.punctuation:
+                if punc in sen:
+                    sen = sen.replace(punc, '')
+
+            tmp = {}
+            tmp['filename'] = name
+            tmp['img_id'] = i
+            tmp['cap_id'] = count
+            tmp['caption'] = sen
+            count += 1
+            caps.append(tmp)
+
+        val_data[i] = caps
+# pk.dump(val_data, open('cub_data/K_val_annotation.pkl', 'w'))
+print 'dump %d in val_data'%len(val_data)
+json.dump(test_data, open('cub_data/K_val_annotation.json', 'w'))

data-prepro/MSCOCO_preprocess/extract_resnet_coco.py

@@ -1,6 +1,10 @@
 import sys
-sys.path.append('/home/PaulChen/deep-residual-networks/caffe/python')
+# sys.path.append('/home/PaulChen/deep-residual-networks/caffe/python')
+sys.path.append('/home/smelly/projects/show-adapt-and-tell/data-prepro/MSCOCO_preprocess/deep-residual-networks/caffe')
+sys.path.append('/home/smelly/projects/show-adapt-and-tell/data-prepro/MSCOCO_preprocess/deep-residual-networks/caffe/python')
+
 import caffe
+
 import numpy as np
 import argparse
 import cv2
@@ -77,7 +81,7 @@ def extract_image(net, image_file):
 
 def split(split, net, feat_dict):
         print 'load ' + split 
-	img_dir = './coco/'
+	img_dir = '/home/smelly/projects/ic_models/im2txt/im2txt/data/mscoco/raw-data/'
 	img_path = os.path.join(img_dir, split)
 	img_list = os.listdir(img_path)
 	pool5_list = []
@@ -90,7 +94,7 @@ def split(split, net, feat_dict):
 
 if __name__ == '__main__':
 	args = parse_args()
-	caffe_path = os.path.join('/home','PaulChen','caffe','python')
+	# caffe_path = os.path.join('/home','PaulChen','caffe','python')
 
 	print 'caffe setting'
 	caffe.set_mode_gpu()