add more code

BigNoobWasTaken · BigNoobWasTaken · commit f6a88e93ec51 · 2024-01-08T20:35:22.000+07:00
diff --git a/src/tensor_array/activation.py b/src/tensor_array/activation.py
@@ -0,0 +1,12 @@
+from tensor_array.core import Tensor
+from tensor_array.core import zeros
+
+def relu(input):
+    tensor_zeros = zeros(shape = input.shape(), dtype = input.dtype())
+    return (input > tensor_zeros).condition(input, tensor_zeros)
+
+def sigmoid(input):
+    return input.sigmoid()
+
+def softmax(input, dim = 0):
+    return input
diff --git a/src/tensor_array/core/libTensorArray_Core.so b/src/tensor_array/core/libTensorArray_Core.so
diff --git a/src/tensor_array/layers/attention/attention.py b/src/tensor_array/layers/attention/attention.py
@@ -2,10 +2,11 @@
 from .. import Layer
 from ..util import Linear
 from tensor_array.core import Tensor
+from tensor_array.activation import softmax
 
 def scaled_dot_product_attention(q, k, v, mask = None):
     attn_scores = q @ k.transpose(len(k.shape()) - 2, len(k.shape()) - 1)
-    attn_probs = SoftMax(attn_scores, len(attn_scores.shape()) - 1)
+    attn_probs = softmax(attn_scores, len(attn_scores.shape()) - 1)
     return attn_probs @ v
 
 class MultiheadAttention(Layer):
diff --git a/src/tensor_array/layers/attention/transformer.py b/src/tensor_array/layers/attention/transformer.py
@@ -1,6 +1,7 @@
 from typing import Any
 from .. import Layer
 from .attention import MultiheadAttention
+from tensor_array.activation import relu
 from ..util import Sequential
 from ..util import Linear
 from ..util import Activation
@@ -9,7 +10,7 @@ class TransformerEncoderImpl(Layer):
     def __init__(self, d_model, n_head, ff_size) -> None:
         self.feed_forward = Sequential([
             Linear(ff_size),
-            Activation(ReLU),
+            Activation(relu),
             Linear(d_model)
         ])
         self.multihead_attn = MultiheadAttention(d_model, n_head)
diff --git a/src/tensor_array/layers/normalization/__init__.py b/src/tensor_array/layers/normalization/__init__.py
@@ -0,0 +1 @@
+from tensor_array.layers.normalization import Normalization
diff --git a/src/tensor_array/layers/normalization/normalization.py b/src/tensor_array/layers/normalization/normalization.py
@@ -0,0 +1,4 @@
+from .. import Layer
+
+class Normalization(Layer):
+    pass
diff --git a/tensor-array-repo/Tensor-Array b/tensor-array-repo/Tensor-Array
@@ -1 +1 @@
-Subproject commit 3257b48eb67365c1010f7bb7cb32fe3b752e4578
+Subproject commit 099aa87fdc564f97169cadbcb455016fe27b64e9

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+from tensor_array.layers.normalization import Normalization`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +from .. import Layer
++
 +class Normalization(Layer):
 +    pass