A lot of changes, refactorize MPO and building ML core

core/model.py

@@ -0,0 +1,148 @@
+from numpy.linalg.linalg import matrix_rank
+from syngular.tensor.tensor_train import MatrixProductOperator, MatrixProductState
+
+import numpy as np
+from opt_einsum import contract
+
+class Model:
+
+    def __init__(self, layers):
+        self.layers = layers
+
+    def predict(self, inputs):
+        values = inputs
+        for layer in self.layers:
+            values = layer(values)
+            # print("Values shape", values._shape)
+
+        return values
+
+    def build(self):
+        for layer in self.layers:
+            if not layer.is_built:
+                layer.build(None)
+                layer.is_built = True
+
+    def train(self):
+        pass
+
+    def draw(self):
+        repr = ''
+        for layer in self.layers:
+            repr += layer.draw()
+        return repr
+
+
+class Layer:
+
+    def __init__(self):
+
+        self.trainable_tensor_weights = []
+        self.trainable_tensor_bias = []
+
+        self.is_built = False
+
+    def __call__(self, inputs):
+        input_shape = inputs.shape
+        if not self.is_built:
+            self.build(input_shape)
+            self.is_built = True
+        else:
+            print("Built")
+
+    def build(self, input_shape):
+        pass
+
+    def draw(self):
+        repr = ''
+        for weight in self.trainable_tensor_weights:
+            mp = weight["weight"]
+            repr += "\t"+"|   " * mp.sites_number + "\n"
+            repr += "\t"+("O---" * (mp.sites_number-1)) + "O" + "\n"
+            repr += "\t"+"|   " * mp.sites_number + "\n"
+        return repr
+
+
+    def add_weight(self, input_shape, output_shape, bond, name=None, initializer="normal"):
+        if name == None:
+            name = f'weight_{np.random.randint(0,999999)}'
+
+        if initializer == "normal":
+            weight = np.random.normal(size=(*self._input_shape, *self._output_shape))
+        else:
+            weight = np.zeros(shape=(*self._input_shape, *self._output_shape))
+
+        matrix_product_weight = MatrixProductOperator(weight, bond_shape=bond)
+        matrix_product_weight.decompose(mode='left')
+
+        self.trainable_tensor_weights.append({'name': name, 'weight': matrix_product_weight})
+
+
+
+    def add_bias(self, size, name=None, initializer="normal"):
+        if name == None:
+            name = f'bias_{np.random.randint(0,999999)}'
+
+        if initializer == "normal":
+            bias = np.random.normal(size=size)
+        else:
+            bias = np.zeros(shape=size)
+
+        self.trainable_tensor_bias.append({name: name, bias: bias})
+
+class Linear(Layer):
+
+    def __init__(self, 
+        input_units, output_units,
+        core=1, bond=None,
+        bias_initializer="normal",
+        weights_initializer="normal",
+        activation="relu"):
+
+        super(Linear, self).__init__()
+
+        self.input_units = input_units
+        self.output_units = output_units
+
+        self.core = core
+        self.bond_dimension = bond
+
+        self.input_core_dim = round(self.input_units**(1/self.core))
+        self.output_core_dim = round(self.output_units**(1/self.core))
+
+        self._input_shape    = (self.input_core_dim,) * self.core
+        self._output_shape   = (self.output_core_dim,) * self.core
+        self._bond_shape     = (self.bond_dimension,) * (self.core-1)
+
+        self.bias_initializer = bias_initializer
+        self.weights_initializer = weights_initializer
+
+        self.activation = activation
+
+    def build(self, input_shape):
+        # self.add_bias(self._output_shape, name="bias", initializer="normal")
+        # print(self._input_shape, self._output_shape)
+        self.add_weight(self._input_shape, self._output_shape, bond=self._bond_shape, name="bias", initializer="normal")
+
+    def __call__(self, inputs):
+        super(Linear, self).__call__(inputs)
+
+        weight = self.trainable_tensor_weights[0]["weight"]
+
+        print("Weight", weight)
+        print("input", inputs)
+        print("contract", MatrixProductOperator.contract(weight, inputs))
+
+        return MatrixProductOperator.contract(weight, inputs)
+
+class Output(Layer):
+
+    def __init__(self, output_shape):
+        super(Output, self).__init__()
+
+        self.output_shape = output_shape
+
+    def __call__(self, inputs):
+        # print(">", inputs)
+
+        return inputs#.reshape(self.output_shape)

documentation/matrix_product/MatrixProduct.md

@@ -0,0 +1,46 @@
+# Matrix Product
+
+
+Importation 
+```python
+from syngular.tensor.tensor_train import MatrixProductState
+from syngular.tensor.tensor_train import MatrixProductOperator
+```
+
+### Addition
+
+```python
+tensor = np.arange(0,16).reshape((2,2,2,2))
+
+X = MatrixProductOperator(tensor, bond_shape=(4,))
+Y = MatrixProductOperator(tensor, bond_shape=(4,))
+
+Z = X + Y
+```
+
+### Multiplication
+
+```python
+tensor = np.arange(0,16).reshape((2,2,2,2))
+tensor2 = np.arange(0,4).reshape((2,2))
+
+X = MatrixProductOperator(tensor, bond_shape=(4,))
+Y = MatrixProductOperator(tensor, bond_shape=(4,))
+Z = MatrixProductOperator(tensor, bond_shape=(4,))
+S = MatrixProductState(tensor2, bond_shape=(4,))
+
+W = S @ ((X @ Y) + (Y @ X)) @ S
+```
+
+![Diagram](tensor.jpg)
+
+### Compression
+
+
+```python
+tensor = np.arange(0,16).reshape((2,2,2,2))
+
+X = MatrixProductOperator(tensor, bond_shape=(4,))
+
+W = X >> 2
+```

layers/TensorDense.py

@@ -9,6 +9,8 @@
 import tensornetwork as tn
 import numpy as np
 
+from opt_einsum import contract
+
 def unfold_dim(shape):
     return reduce(lambda x, y: x*y, shape)
 
@@ -40,9 +42,9 @@ def build(self, tt_input_shape):
 
             self.tt_input_shape = tuple([roots] * self.cores_number)
 
-            print("reshaped", tt_input_shape)
+            #print("reshaped", tt_input_shape)
 
-        print("Input shape", self.tt_input_shape)
+        #print("Input shape", self.tt_input_shape)
 
         self.bias = tf.Variable(tf.zeros(shape=self.tt_output_shape), name="bias", trainable=True)
 
@@ -76,31 +78,49 @@ def build(self, tt_input_shape):
     def call(self, inputs):
 
         def process(input, bias):
-            input_reshaped = tf.reshape(input,self.tt_input_shape)
+            # input_reshaped = tf.reshape(input,self.tt_input_shape)
 
-            last_idx = self.cores_number-1
-            einsum_structure = []
+            # last_idx = self.cores_number-1
+            # einsum_structure = []
 
-            cores = [tn.Node(core, backend="tensorflow").tensor for core in self.cores]
-            x = tn.Node(input_reshaped, backend="tensorflow")
+            # cores = [tn.Node(core, backend="tensorflow").tensor for core in self.cores]
+            # x = tn.Node(input_reshaped, backend="tensorflow")
 
 
-            einsum_structure = []
-            einsum_structure.append(list(range(1, self.cores_number+1)))
-            einsum_structure.append([1, -(self.cores_number+1), 2*self.cores_number+1])
+            # einsum_structure = []
+            # einsum_structure.append(list(range(1, self.cores_number+1)))
+            # einsum_structure.append([1, -(self.cores_number+1), 2*self.cores_number+1])
 
-            for idx in range(1, last_idx):
-                einsum_structure.append([idx+1, -(self.cores_number+idx+1), 2*self.cores_number+idx+1])
+            # for idx in range(1, last_idx):
+            #     einsum_structure.append([idx+1, -(self.cores_number+idx+1), 2*self.cores_number+idx+1])
 
-            einsum_structure.append([last_idx+1, -(self.cores_number+last_idx+1), 2*self.cores_number+last_idx-1+1])
+            # einsum_structure.append([last_idx+1, -(self.cores_number+last_idx+1), 2*self.cores_number+last_idx-1+1])
 
-            print(einsum_structure)
+            # #print(einsum_structure)
+
+            # result = tn.ncon(
+            #     tensors = [x.tensor] + cores,
+            #     network_structure = einsum_structure,
+            #     backend = "tensorflow"
+            # )
+
+            x = tf.reshape(input,self.tt_input_shape)#tf.reshape(input, ((self.tt_input_shape[0],)*self.cores_number))
+
+
+            struct = []
+            struct += [x, list(range(1, self.cores_number+1))]
+            struct += [self.cores[0], [1, self.cores_number+1, 2*self.cores_number+1]]
+
+            for idx in range(2, self.cores_number):
+                struct += [self.cores[idx-1]]
+                struct += [[idx, self.cores_number+idx, 2*self.cores_number+1, 2*self.cores_number+2]]
+
+            struct += [self.cores[-1], [self.cores_number, 2*self.cores_number, 3*self.cores_number-1]]
+            struct += [list(range(self.cores_number+1, 2*self.cores_number+1))]
+
+
+            result = contract(*struct)
 
-            result = tn.ncon(
-                tensors = [x.tensor] + cores,
-                network_structure = einsum_structure,
-                backend = "tensorflow"
-            )
             # print(type(self.cores[0].numpy()))
 
             # cores = list(map(lambda core: tf.convert_to_tensor(core), self.cores))

layers/TensorRNN.py

@@ -84,3 +84,12 @@ def process(input):
         result = tf.vectorized_map(lambda vec: process(vec, self.bias), inputs)
         return self.activation(tf.reshape(result, (-1, self.tt_output_shape_unfold)))
 
+
+if __name__ == "__main__":
+
+    from keras.layers import SimpleRNN
+    from keras import Sequential
+
+    model = Sequential([
+        SimpleRNN()
+    ])