From fa0567ac4d1cf6d278eaede14afa33f3aa194655 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:03:57 -0700 Subject: [PATCH 01/64] Create README.md --- keras implementation/ README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 keras implementation/ README.md diff --git a/keras implementation/ README.md b/keras implementation/ README.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/keras implementation/ README.md @@ -0,0 +1 @@ + From d834d30e48bf23e3076b62799fc147e6f65d5674 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:05:51 -0700 Subject: [PATCH 02/64] Create readme --- keras implementation/data/readme | 1 + 1 file changed, 1 insertion(+) create mode 100644 keras implementation/data/readme diff --git a/keras implementation/data/readme b/keras implementation/data/readme new file mode 100644 index 0000000..1f28342 --- /dev/null +++ b/keras implementation/data/readme @@ -0,0 +1 @@ +Contains the VQA dataset in txt files From 0d631b6b002f98d13a483303ce8e11cdc2869bc1 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:32:05 -0700 Subject: [PATCH 03/64] Create readme --- keras implementation/creating_models/readme | 1 + 1 file changed, 1 insertion(+) create mode 100644 keras implementation/creating_models/readme diff --git a/keras implementation/creating_models/readme b/keras implementation/creating_models/readme new file mode 100644 index 0000000..a83201e --- /dev/null +++ b/keras implementation/creating_models/readme @@ -0,0 +1 @@ +Scripts for creating various models From 7738add242273f167ccaa6287ec97be2957cfa41 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:43:18 -0700 Subject: [PATCH 04/64] Add files via upload --- .../creating_models/create_normal_gru.py | 44 ++++++++++++++++++ .../creating_models/create_normal_lstm.py | 45 +++++++++++++++++++ .../create_timedistributedGRU.py | 42 +++++++++++++++++ .../create_timedistributed_lstm.py | 43 ++++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 keras implementation/creating_models/create_normal_gru.py create mode 100644 keras implementation/creating_models/create_normal_lstm.py create mode 100644 keras implementation/creating_models/create_timedistributedGRU.py create mode 100644 keras implementation/creating_models/create_timedistributed_lstm.py diff --git a/keras implementation/creating_models/create_normal_gru.py b/keras implementation/creating_models/create_normal_gru.py new file mode 100644 index 0000000..401e60a --- /dev/null +++ b/keras implementation/creating_models/create_normal_gru.py @@ -0,0 +1,44 @@ +from keras.layers import Dense,Activation,Dropout,Reshape,Merge +from keras.layers.wrappers import TimeDistributed +from keras.layers.recurrent import LSTM,GRU +from keras.optimizers import SGD +from keras.callbacks import ModelCheckpoint,TensorBoard +from keras.models import Sequential,model_from_json + + + +##creating the model in Keras +img_dim=4096 +quest_dim=500 +total_epochs=100 +SEQ_LENGTH=23 +word_vec_dim=300 +hidden_dim_feed=1000 + +######################################################################### + +##will load the image model here +image_model=Sequential() +# image_model.add(Reshape(input_shape = (img_dim,), dims=(img_dim,))) +image_model.add(Reshape((img_dim,), input_shape=(img_dim,))) + +##will import the language model here +questions_model=Sequential() +questions_model.add(GRU(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=True)) +# questions_model.add(GRU(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=False)) + + +################################################################################ + +model=Sequential() +##merging image_dim and quest_dim +model.add(Merge([image_model,questions_model],mode='concat')) +model.add(Dense(output_dim=hidden_dim_feed,activation='relu',init='uniform')) +model.add(Dense(output_dim=500,activation='relu',init='uniform')) +model.add(Dense(output_dim=1000,activation='softmax')) +##because we are choosing 1000 most frequently occuring questions +# ##saving the model to json +name_model_mlp=model.to_json() +open('C:/Users/ezio/Desktop/vqa/model/normal_2_lstm_nodistributed_2hidden.json','w').write(name_model_mlp) +print("Model is saved") + diff --git a/keras implementation/creating_models/create_normal_lstm.py b/keras implementation/creating_models/create_normal_lstm.py new file mode 100644 index 0000000..772bd51 --- /dev/null +++ b/keras implementation/creating_models/create_normal_lstm.py @@ -0,0 +1,45 @@ +from keras.layers import Dense,Activation,Dropout,Reshape,Merge +from keras.layers.wrappers import TimeDistributed +from keras.layers.recurrent import LSTM,GRU +from keras.optimizers import SGD +from keras.callbacks import ModelCheckpoint,TensorBoard +from keras.models import Sequential,model_from_json + + + +##creating the model in Keras +img_dim=4096 +quest_dim=500 +total_epochs=100 +SEQ_LENGTH=23 +word_vec_dim=300 +hidden_dim_feed=1000 + +######################################################################### + +##will load the image model here +image_model=Sequential() +# image_model.add(Reshape(input_shape = (img_dim,), dims=(img_dim,))) +image_model.add(Reshape((img_dim,), input_shape=(img_dim,))) + +##will import the language model here +questions_model=Sequential() +questions_model.add(LSTM(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=True)) +questions_model.add(LSTM(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=False)) + + +################################################################################ + +model=Sequential() +##merging image_dim and quest_dim +model.add(Merge([image_model,questions_model],mode='concat')) +model.add(Dense(output_dim=hidden_dim_feed,activation='relu',init='uniform')) +model.add(Dense(output_dim=500,activation='relu',init='uniform')) +model.add(Dense(output_dim=1000,activation='softmax')) +##because we are choosing 1000 most frequently occuring questions + +# ##saving the model to json +name_model_mlp=model.to_json() +open('C:/Users/ezio/Desktop/vqa/model/normal_2_lstm_nodistributed_2hidden.json','w').write(name_model_mlp) +print("Model is saved") + diff --git a/keras implementation/creating_models/create_timedistributedGRU.py b/keras implementation/creating_models/create_timedistributedGRU.py new file mode 100644 index 0000000..409ca27 --- /dev/null +++ b/keras implementation/creating_models/create_timedistributedGRU.py @@ -0,0 +1,42 @@ +from keras.layers import Dense,Activation,Dropout,Reshape,Merge +from keras.layers.wrappers import TimeDistributed +from keras.layers.recurrent import LSTM,GRU +from keras.optimizers import SGD +from keras.callbacks import ModelCheckpoint,TensorBoard +from keras.models import Sequential,model_from_json + + +img_dim=4096 +quest_dim=50 +total_epochs=100 +SEQ_LENGTH=23 +word_vec_dim=300 +hidden_dim_feed=1000 + +######################################################################### +# ''' +##will load the image model here +image_model=Sequential() +# image_model.add(Reshape(input_shape = (img_dim,), dims=(img_dim,))) +image_model.add(Reshape((img_dim,), input_shape=(img_dim,))) + +##will import the language model here +questions_model=Sequential() +questions_model.add(GRU(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=True)) +questions_model.add(TimeDistributed(Dense(100,activation='relu',init='uniform'))) +questions_model.add(Reshape((23*100,))) + +################################################################################ + +model=Sequential() +##merging image_dim and quest_dim +model.add(Merge([image_model,questions_model],mode='concat')) +model.add(Dense(output_dim=hidden_dim_feed,activation='relu',init='uniform')) +model.add(Dense(output_dim=500,activation='relu',init='uniform')) +model.add(Dense(output_dim=1000,activation='softmax')) + +# ##saving the model to json +name_model_mlp=model.to_json() +open('C:/Users/ezio/Desktop/vqa/model/normal_1_gru_timedistributed_2hidden.json','w').write(name_model_mlp) +print("Model is saved") +import gc; gc.collect() diff --git a/keras implementation/creating_models/create_timedistributed_lstm.py b/keras implementation/creating_models/create_timedistributed_lstm.py new file mode 100644 index 0000000..60d2489 --- /dev/null +++ b/keras implementation/creating_models/create_timedistributed_lstm.py @@ -0,0 +1,43 @@ +from keras.layers import Dense,Activation,Dropout,Reshape,Merge +from keras.layers.wrappers import TimeDistributed +from keras.layers.recurrent import LSTM,GRU +from keras.optimizers import SGD +from keras.callbacks import ModelCheckpoint,TensorBoard +from keras.models import Sequential,model_from_json + + +img_dim=4096 +quest_dim=50 +total_epochs=100 +SEQ_LENGTH=23 +word_vec_dim=300 +hidden_dim_feed=1000 + +######################################################################### +# ''' +##will load the image model here +image_model=Sequential() +# image_model.add(Reshape(input_shape = (img_dim,), dims=(img_dim,))) +image_model.add(Reshape((img_dim,), input_shape=(img_dim,))) + +##will import the language model here +questions_model=Sequential() +questions_model.add(LSTM(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=True)) +# questions_model.add(LSTM(output_dim=quest_dim,input_shape=(SEQ_LENGTH,word_vec_dim),return_sequences=False)) +questions_model.add(TimeDistributed(Dense(100,activation='relu',init='uniform'))) +questions_model.add(Reshape((23*100,))) + +################################################################################ + +model=Sequential() +##merging image_dim and quest_dim +model.add(Merge([image_model,questions_model],mode='concat')) +model.add(Dense(output_dim=hidden_dim_feed,activation='relu',init='uniform')) +model.add(Dense(output_dim=500,activation='relu',init='uniform')) +model.add(Dense(output_dim=1000,activation='softmax')) + +# ##saving the model to json +name_model_mlp=model.to_json() +open('C:/Users/ezio/Desktop/vqa/model/normal_1_gru_timedistributed_2hidden.json','w').write(name_model_mlp) +print("Model is saved") +import gc; gc.collect() From ad5d0a952253a4831afd016627fa4d6fb3a18493 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:45:06 -0700 Subject: [PATCH 05/64] Create readme --- keras implementation/pre-trained models/readme | 1 + 1 file changed, 1 insertion(+) create mode 100644 keras implementation/pre-trained models/readme diff --git a/keras implementation/pre-trained models/readme b/keras implementation/pre-trained models/readme new file mode 100644 index 0000000..08a3e47 --- /dev/null +++ b/keras implementation/pre-trained models/readme @@ -0,0 +1 @@ +Collection of pretrained models with their weights From 4fbf89172199d0c3cc470006cc192ef946a8f9c2 Mon Sep 17 00:00:00 2001 From: fenildoshi Date: Tue, 23 May 2017 11:49:34 -0700 Subject: [PATCH 06/64] Create readme --- .../pre-trained models/LSTM_Time Distributed/readme | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 keras implementation/pre-trained models/LSTM_Time Distributed/readme diff --git a/keras implementation/pre-trained models/LSTM_Time Distributed/readme b/keras implementation/pre-trained models/LSTM_Time Distributed/readme new file mode 100644 index 0000000..d669da6 --- /dev/null +++ b/keras implementation/pre-trained models/LSTM_Time Distributed/readme @@ -0,0 +1,3 @@ +##Architecture + +Here the word vector of every word is collected in a 3-d Tensor-(None,