Add skip connections and normalize.

Author: sseto001

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/EdgeAttention/Architecture/EdgeAttention.json

@@ -121,10 +121,15 @@
               "inputs": [ "reduce", "RB" ],
               "gradientResultTo": [ null, "DRB" ]
             },
+            {
+              "id": "reduce_add_skip",
+              "type": "BatchAddThreeOperation",
+              "inputs": [ "reduce_add", "keys_edgeFeatures", "values_edgeFeatures" ]
+            },
             {
               "id": "layernorm_reduce",
               "type": "BatchLayerNormalizationOperation",
-              "inputs": [ "reduce_add" ]
+              "inputs": [ "reduce_add_skip" ]
             },
             {
               "id": "output_act",

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/GCN/Architecture/MessagePassing.json

@@ -52,7 +52,12 @@
             {
               "id": "ah_w_act",
               "type": "BatchLeakyReLUOperation",
-              "inputs": [ "ah_w_broadcasting" ],
+              "inputs": [ "ah_w_broadcasting" ]
+            },
+            {
+              "id": "ah_w_act_softmax",
+              "type": "BatchSineSoftmaxOperation",
+              "inputs": [ "ah_w_act" ],
               "setResultTo": "Output"
             }
           ]

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/Readout/Architecture/Readout.json

@@ -204,10 +204,15 @@
               "inputs": [ "reduce", "RB" ],
               "gradientResultTo": [ null, "DRB" ]
             },
+            {
+              "id": "reduce_add_skip",
+              "type": "BatchMatrixAddThreeOperation",
+              "inputs": [ "reduce_add", "keys_pathFeatures", "values_pathFeatures" ]
+            },
             {
               "id": "layernorm_reduce",
               "type": "BatchLayerNormalizationOperation",
-              "inputs": [ "reduce_add" ]
+              "inputs": [ "reduce_add_skip" ]
             },
             {
               "id": "output_act",
@@ -226,7 +231,22 @@
         {
           "id": "output_avg",
           "type": "BatchMatrixAverageOperation",
-          "inputs": [ "pre_output" ],
+          "inputs": [ "pre_output" ]
+        },
+        {
+          "id": "output_avg_trans",
+          "type": "MatrixTransposeOperation",
+          "inputs": [ "output_avg" ]
+        },
+        {
+          "id": "output_avg_softmax",
+          "type": "BatchSineSoftmaxOperation",
+          "inputs": [ "output_avg_trans" ]
+        },
+        {
+          "id": "output_avg_softmax_trans",
+          "type": "MatrixTransposeOperation",
+          "inputs": [ "output_avg_softmax" ],
           "setResultTo": "Output"
         }
       ]

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/Transformer/Architecture/Transformer.json

@@ -217,10 +217,15 @@
               "inputs": [ "reduce", "RB" ],
               "gradientResultTo": [ null, "DRB" ]
             },
+            {
+              "id": "reduce_add_skip",
+              "type": "MatrixAddThreeOperation",
+              "inputs": [ "reduce_add", "keys_pathFeatures", "values_pathFeatures" ]
+            },
             {
               "id": "layernorm_reduce",
               "type": "BatchLayerNormalizationOperation",
-              "inputs": [ "reduce_add" ]
+              "inputs": [ "reduce_add_skip" ]
             },
             {
               "id": "output_act",