Extracted some hidden files

models/cross-dimensional-attention/README.md

@@ -0,0 +1,24 @@
+# Cross-Dimensional-Attention Model
+
+## [CDSA: Cross-Dimensional Self-Attention for Multivariate, Geo-tagged Time Series Imputation](https://arxiv.org/pdf/1905.09904.pdf)
+_Jiawei Ma, Zheng Shou, Alireza Zareian, Hassan Mansour, Anthony Vetro, Shih-Fu Chang_
+
+Many real-world applications involve multivariate, geo-tagged time series data: at
+each location, multiple sensors record corresponding measurements. For example,
+air quality monitoring system records PM2.5, CO, etc. The resulting time-series
+data often has missing values due to device outages or communication errors. In
+order to impute the missing values, state-of-the-art methods are built on Recurrent
+Neural Networks (RNN), which process each time stamp sequentially, prohibiting
+the direct modeling of the relationship between distant time stamps. Recently, the
+self-attention mechanism has been proposed for sequence modeling tasks such as
+machine translation, significantly outperforming RNN because the relationship between each two time stamps can be modeled explicitly. In this paper, we are the first
+to adapt the self-attention mechanism for multivariate, geo-tagged time series data.
+In order to jointly capture the self-attention across multiple dimensions, including
+time, location and the sensor measurements, while maintain low computational
+complexity, we propose a novel approach called Cross-Dimensional Self-Attention
+(CDSA) to process each dimension sequentially, yet in an order-independent manner. Our extensive experiments on four real-world datasets, including three standard
+benchmarks and our newly collected NYC-traffic dataset, demonstrate that our
+approach outperforms the state-of-the-art imputation and forecasting methods. A
+detailed systematic analysis confirms the effectiveness of our design choices.
+
+![image](https://drive.google.com/uc?export=view&id=1qJzGTBewD-Q9gT7M1FaKqAKW7H6S4OBK)

models/cross-dimensional-attention/tensorflow/Base_Handler.py

@@ -0,0 +1,82 @@
+# uncompyle6 version 3.7.0
+# Python bytecode 2.7 (62211)
+# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04) 
+# [GCC 8.4.0]
+# Embedded file name: /home/jiawei/Tensor_MultiDim_NYC/Model/Base_Handler.py
+# Compiled at: 2019-04-29 02:51:25
+import tensorflow as tf, numpy as np, yaml, os, h5py, time, sys, math
+from Lib.Model_Visualization import *
+
+class Basement_Handler(object):
+
+    def __init__(self, sess, model_config, is_training):
+        self.sess = sess
+        self.model_config = model_config
+        self.max_grad_norm = float(model_config.get('max_grad_norm', 5.0))
+        self.init_logging(is_training)
+        self.logger.info(model_config)
+
+    def init_logging(self, is_training):
+        if is_training is not True:
+            base_dir = self.model_config.get('result_data')
+            folder_dir = self.model_config.get('category') + '_for_' + self.model_config.get('data_name')
+        else:
+            base_dir = self.model_config.get('result_model')
+            folder_dir = generate_folder_id(self.model_config)
+        log_dir = os.path.join(self.model_config.get('result_dir'), base_dir, folder_dir)
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+        self.log_dir = log_dir
+        self.logger = get_logger(self.log_dir, folder_dir)
+        self.writer = tf.summary.FileWriter(self.log_dir)
+
+    def trainable_parameter_info(self):
+        total_parameters = 0
+        for variable in tf.trainable_variables():
+            total_parameters += np.product([ x.value for x in variable.get_shape() ])
+
+        self.logger.info('Total number of trainable parameters: %d' % total_parameters)
+        for var in tf.global_variables():
+            self.logger.debug('%s, %s' % (var.name, var.get_shape()))
+
+    def summary_logging(self, global_step, names, values):
+        for name, value in zip(names, values):
+            summary = tf.Summary()
+            summary_value = summary.value.add()
+            summary_value.simple_value = value
+            summary_value.tag = name
+            self.writer.add_summary(summary, global_step)
+
+    def save_model(self, saver, epoch, val_loss):
+        config_filename = 'config_%02d.yaml' % epoch
+        config = dict(self.model_config)
+        global_step = self.sess.run(tf.train.get_or_create_global_step())
+        config['epoch'] = epoch
+        config['global_step'] = global_step
+        config['log_dir'] = self.log_dir
+        config['model_filename'] = saver.save(self.sess, os.path.join(self.log_dir, 'models-%.4f' % val_loss), global_step=global_step, write_meta_graph=False)
+        with open(os.path.join(self.log_dir, config_filename), 'w') as (f):
+            yaml.dump(config, f)
+        return config['model_filename']
+
+    def restore(self):
+        config = dict(self.model_config)
+        self.pred_step = config['global_step']
+        model_filename = config['model_filename']
+        saver = tf.train.Saver(tf.global_variables())
+        saver.restore(self.sess, model_filename)
+
+    def train_test_valid_assignment(self):
+        pass
+
+    def initial_parameter(self):
+        pass
+
+    def data_assignment(self):
+        pass
+
+    def train(self):
+        pass
+
+    def test(self):
+        pass

models/cross-dimensional-attention/tensorflow/Base_TFModel.py

@@ -0,0 +1,36 @@
+# uncompyle6 version 3.7.0
+# Python bytecode 2.7 (62211)
+# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04) 
+# [GCC 8.4.0]
+# Embedded file name: /home/jiawei/Tensor_MultiDim_NYC/Model/Base_TFModel.py
+# Compiled at: 2019-04-29 02:51:25
+import os, tensorflow as tf
+
+class Basement_TFModel(object):
+    """Define and Initialize the basic/necessary element of a tensorflow model """
+    __module__ = __name__
+
+    def __init__(self, sess, config, learning_rate, is_training):
+        self.sess = sess
+        self.config = config
+        self.is_training = is_training
+        self.model_name = config.get('model_name', 'MDAnalyzer')
+        self.train_op = None
+        self.learning_rate = learning_rate
+        self.max_grad_norm = float(config.get('max_grad_norm', 5.0))
+        self.loss = None
+        self.loss_func = config.get('loss_func', 'RMSE')
+        self.maximum_type = int(config.get('upbound', 1))
+        return
+
+    def set_lr(self, new_learning_rate):
+        self.learning_rate = new_learning_rate
+
+    def save_checkpoint(self, step=None):
+        pass
+
+    def load_checkpoint(self, step=None):
+        pass
+
+    def initial_parameter(self):
+        pass

models/cross-dimensional-attention/tensorflow/BurstLoss_Generation.ipynb

@@ -0,0 +1,187 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/dvmm-filer2/users/jiawei/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
+      "  from ._conv import register_converters as _register_converters\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import os\n",
+    "import h5py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[u'data', u'mask', u'maximum', u'position']\n",
+      "(186, 6624) (186, 6624) (186, 2) 0.9157041\n",
+      "25.585367 0.9157041 25.585367 2.992725 0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "combine_file = h5py.File('data_mask_maximum.h5','r')\n",
+    "print combine_file.keys()\n",
+    "data = combine_file['data'][:]\n",
+    "mask = combine_file['mask'][:]\n",
+    "position = combine_file['position'][:]\n",
+    "maximum = combine_file['maximum'].value\n",
+    "combine_file.close()\n",
+    "print data.shape,mask.shape,position.shape,mask.mean()\n",
+    "print maximum,mask.mean(),data.max(),data.mean(),data.min()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(24290,)\n",
+      "(5887,)\n",
+      "6.350773 9.809643 0.08429594\n"
+     ]
+    }
+   ],
+   "source": [
+    "mask_miss = 1.0-mask\n",
+    "mask_time,mask_delta = np.zeros_like(mask),np.zeros_like(mask)\n",
+    "mask_time[:,0] = mask_miss[:,0]\n",
+    "for cam in range(186):\n",
+    "    for time in range(1,6624):\n",
+    "        if mask_miss[cam,time] == 1:\n",
+    "            mask_time[cam,time] = mask_time[cam,time-1]+1\n",
+    "    mask_delta[cam,:] = mask_time[cam,:]\n",
+    "    while(time>0):\n",
+    "        if mask_time[cam,time] > 1:\n",
+    "            length = int(mask_time[cam,time]-1)\n",
+    "            mask_time[cam,time-length:time] = mask_time[cam,time]\n",
+    "            mask_delta[cam,time-length:time] = 0\n",
+    "            time -= length\n",
+    "        time -= 1\n",
+    "delta_seq = mask_delta[mask_delta != 0]\n",
+    "delta_seq = np.sort(delta_seq)\n",
+    "delta_burst = delta_seq[delta_seq<=134]\n",
+    "print delta_burst.shape\n",
+    "delta_burst = delta_burst[delta_burst > 1]\n",
+    "print delta_burst.shape\n",
+    "print delta_burst.mean(),delta_burst.std(),mask_miss.mean()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "missing rate: 0.9009605050086975\n"
+     ]
+    }
+   ],
+   "source": [
+    "burst_mean,burst_std,mask_burst,ratio = delta_burst.mean(),delta_burst.std(),np.zeros_like(mask_miss),0.618\n",
+    "for cam in range(186):\n",
+    "    time = 0\n",
+    "    while(time<6624):\n",
+    "        if np.random.uniform(size=1) < ratio:\n",
+    "            length = int(np.random.normal(burst_mean, burst_std, 1))\n",
+    "            if length > 0:\n",
+    "                mask_burst[cam,time:time+length] = 1.0\n",
+    "                time += length\n",
+    "            else:\n",
+    "                time += 1\n",
+    "        else:\n",
+    "            time += 1\n",
+    "move = mask_burst+mask_miss\n",
+    "move[move>1] = 1\n",
+    "move = 1.0-move\n",
+    "for i in range(6624-48*12+1):\n",
+    "    if move[:,i:i+48*12].mean() == 0:\n",
+    "        print i\n",
+    "print 'missing rate:', 1.0-move.mean()\n",
+    "\n",
+    "combine_file = h5py.File('burst_90.h5','w')\n",
+    "combine_file['mask'] = move\n",
+    "combine_file.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.7999438 1.0 0.0\n",
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "combine_file = h5py.File('burst_20.h5','r')\n",
+    "move = combine_file['mask'][:]\n",
+    "combine_file.close()\n",
+    "print move.mean(),move.max(),move.min()\n",
+    "for i in range(6624-48*12+1):\n",
+    "    if move[:,i:i+48*12].mean() == 0:\n",
+    "        print i\n",
+    "temp = move[move!=0]\n",
+    "\n",
+    "print temp[temp!=1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

models/cross-dimensional-attention/tensorflow/Config.yaml

@@ -0,0 +1,55 @@
+---
+# === Configure Main function === #
+category: Traffic-NYC
+data_name: data_mask_maximum
+mask_name: burst_70
+meas_index: 1
+model_name: MDAnalyzer
+result_dir: Result
+result_model: Model-Config
+result_data: Validation-Result
+GPU: 1
+# === Configure Model Handler === #
+# Data Assignment
+batch_size: 19
+period_enc: 432
+period_dec: 432
+
+# Model training Controler
+epoch_iter: 600
+epochs: 100
+patience: 50
+learning_rate: 0.001
+lr_decay: 0.3
+lr_decay_epoch: 20
+lr_decay_interval: 15
+loss_func: RMSE
+upbound: 1
+max_grad_norm: 5.0
+
+# === Configure in MDAnalyzer Model pipeline === #
+# Frame: Sequence Element-wise-addition Concatenation Dimension-reduce Independent
+model_structure: Sequence
+num_enclayer: 7
+num_declayer: 7
+num_heads: 12
+units_IDw: 14
+units_Timew: 6
+# Query, Key and Value
+time_stride_AM: 1
+time_stride_V: 1
+time_fuse_AM: 1
+time_fuse_V: 1
+units_value: 3
+units_weight: 12
+drop_rate_attention: 0.0
+# Filter SetUp
+units_conv: 8
+units_pred: 8
+filter_encdec: dense
+filter_pred: dense
+drop_rate_forward: 0.1
+# Mask label
+flag_identity: False
+flag_time: False
+flag_imputation: True