Batch normalization layers adjusted

Author: Sergio0694

NeuralNetwork.NET/Networks/Layers/Abstract/BatchNormalizationLayerBase.cs

@@ -1,7 +1,9 @@
 using System;
 using System.IO;
+using System.Runtime.CompilerServices;
 using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Enums;
+using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Structs;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Helpers;
@@ -30,14 +32,20 @@ internal abstract class BatchNormalizationLayerBase : WeightedLayerBase
         [NotNull]
         public float[] Sigma2 { get; }
 
-        // The current iteration number (for the Cumulative Moving Average)
-        private int _Iteration;
+        /// <summary>
+        /// Gets the current iteration number (for the Cumulative Moving Average)
+        /// </summary>
+        public int Iteration { get; private set; }
 
         /// <summary>
         /// Gets the current CMA factor used to update the <see cref="Mu"/> and <see cref="Sigma2"/> tensors
         /// </summary>
         [JsonProperty(nameof(CumulativeMovingAverageFactor), Order = 6)]
-        public float CumulativeMovingAverageFactor => 1f / (1 + _Iteration);
+        public float CumulativeMovingAverageFactor
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => 1f / (1 + Iteration);
+        }
 
         /// <inheritdoc/>
         public override String Hash => Convert.ToBase64String(Sha256.Hash(Weights, Biases, Mu, Sigma2));
@@ -74,24 +82,26 @@ protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mod
             NormalizationMode = mode;
         }
 
-        protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
+        protected BatchNormalizationLayerBase(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
             : base(shape, shape, w, b, activation)
         {
             if (w.Length != b.Length) throw new ArgumentException("The size for both gamme and beta paarameters must be the same");
             if (mode == NormalizationMode.Spatial && w.Length != shape.Channels ||
                 mode == NormalizationMode.PerActivation && w.Length != shape.Size)
                 throw new ArgumentException("Invalid parameters size for the selected normalization mode");
+            if (iteration < 0) throw new ArgumentOutOfRangeException(nameof(iteration), "The iteration value must be aat least equal to 0");
             if (mu.Length != w.Length || sigma2.Length != w.Length)
                 throw new ArgumentException("The mu and sigma2 parameters must match the shape of the gamma and beta parameters");
             NormalizationMode = mode;
+            Iteration = iteration;
             Mu = mu;
             Sigma2 = sigma2;
         }
 
         /// <inheritdoc/>
         public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         {
-            if (NetworkTrainer.BackpropagationInProgress) ForwardTraining(1f / (1 + _Iteration++), x, out z, out a);
+            if (NetworkTrainer.BackpropagationInProgress) ForwardTraining(1f / (1 + Iteration++), x, out z, out a);
             else ForwardInference(x, out z, out a);
         }
 
@@ -112,11 +122,22 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         /// <param name="a">The output activation on the current layer</param>
         public abstract void ForwardTraining(float factor, in Tensor x, out Tensor z, out Tensor a);
 
+        /// <inheritdoc/>
+        public override bool Equals(INetworkLayer other)
+        {
+            if (!base.Equals(other)) return false;
+            return other is BatchNormalizationLayerBase layer &&
+                   Iteration == layer.Iteration &&
+                   Mu.ContentEquals(layer.Mu) &&
+                   Sigma2.ContentEquals(layer.Sigma2);
+        }
+
         /// <inheritdoc/>
         public override void Serialize(Stream stream)
         {
             base.Serialize(stream);
             stream.Write(NormalizationMode);
+            stream.Write(Iteration);
             stream.Write(Mu.Length);
             stream.WriteShuffled(Mu);
             stream.Write(Sigma2.Length);

NeuralNetwork.NET/Networks/Layers/Cpu/BatchNormalizationLayer.cs

@@ -18,8 +18,8 @@ internal sealed class BatchNormalizationLayer : BatchNormalizationLayerBase
         public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, ActivationType activation)
             : base(shape, mode, activation) { }
 
-        public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
-            : base(shape, mode, w, b, mu, sigma2, activation) { }
+        public BatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
+            : base(shape, mode, w, b, iteration, mu, sigma2, activation) { }
 
         #region Implementation
 
@@ -98,14 +98,15 @@ public static INetworkLayer Deserialize([NotNull] Stream stream)
             if (!stream.TryRead(out int bLength)) return null;
             float[] biases = stream.ReadUnshuffled(bLength);
             if (!stream.TryRead(out NormalizationMode mode)) return null;
+            if (!stream.TryRead(out int iteration)) return null;
             if (!stream.TryRead(out int mLength)) return null;
             float[] mu = stream.ReadUnshuffled(mLength);
             if (!stream.TryRead(out int sLength)) return null;
             float[] sigma2 = stream.ReadUnshuffled(sLength);
-            return new BatchNormalizationLayer(input, mode, weights, biases, mu, sigma2, activation);
+            return new BatchNormalizationLayer(input, mode, weights, biases, iteration, mu, sigma2, activation);
         }
 
         /// <inheritdoc/>
-        public override INetworkLayer Clone() => new BatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
+        public override INetworkLayer Clone() => new BatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Iteration, Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
     }
 }

NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnBatchNormalizationLayer.cs

@@ -52,8 +52,8 @@ public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode,
             SetupCuDnnInfo();
         }
 
-        public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b,  [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
-            : base(shape, mode, w, b, mu, sigma2, activation)
+        public CuDnnBatchNormalizationLayer(in TensorInfo shape, NormalizationMode mode, [NotNull] float[] w, [NotNull] float[] b, int iteration, [NotNull] float[] mu, [NotNull] float[] sigma2, ActivationType activation) 
+            : base(shape, mode, w, b, iteration, mu, sigma2, activation)
         {
             Tensor.NewZeroed(1, Mu.Length, out SaveMean);
             Tensor.NewZeroed(1, Mu.Length, out SaveInvVariance);
@@ -158,15 +158,16 @@ public static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
             if (!stream.TryRead(out int bLength)) return null;
             float[] biases = stream.ReadUnshuffled(bLength);
             if (!stream.TryRead(out NormalizationMode mode)) return null;
+            if (!stream.TryRead(out int iteration)) return null;
             if (!stream.TryRead(out int mLength)) return null;
             float[] mu = stream.ReadUnshuffled(mLength);
             if (!stream.TryRead(out int sLength)) return null;
             float[] sigma2 = stream.ReadUnshuffled(sLength);
-            return new CuDnnBatchNormalizationLayer(input, mode, weights, biases, mu, sigma2, activation);
+            return new CuDnnBatchNormalizationLayer(input, mode, weights, biases, iteration, mu, sigma2, activation);
         }
 
         /// <inheritdoc/>
-        public override INetworkLayer Clone() => new CuDnnBatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
+        public override INetworkLayer Clone() => new CuDnnBatchNormalizationLayer(InputInfo, NormalizationMode, Weights.AsSpan().Copy(), Biases.AsSpan().Copy(), Iteration, Mu.AsSpan().Copy(), Sigma2.AsSpan().Copy(), ActivationType);
 
         #region IDisposable
 

Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnLayersTest.cs

@@ -176,7 +176,7 @@ public void PerActivationBatchNormalizationForward()
         {
             BatchNormalizationLayerBase
                 cpu = new BatchNormalizationLayer(TensorInfo.Linear(250), NormalizationMode.PerActivation, ActivationType.ReLU),
-                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
+                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
             TestForward(cpu, gpu, 400);
         }
 
@@ -185,7 +185,7 @@ public void PerActivationBatchNormalizationBackward()
         {
             BatchNormalizationLayerBase
                 cpu = new BatchNormalizationLayer(TensorInfo.Linear(250), NormalizationMode.PerActivation, ActivationType.ReLU),
-                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
+                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.PerActivation, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
             TestBackward(cpu, gpu, 400);
         }
 
@@ -194,7 +194,7 @@ public void SpatialBatchNormalizationForward()
         {
             BatchNormalizationLayerBase
                 cpu = new BatchNormalizationLayer(TensorInfo.Volume(12, 12, 13), NormalizationMode.Spatial, ActivationType.ReLU),
-                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
+                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
             TestForward(cpu, gpu, 400);
         }
 
@@ -203,7 +203,7 @@ public void SpatialBatchNormalizationBackward()
         {
             BatchNormalizationLayerBase
                 cpu = new BatchNormalizationLayer(TensorInfo.Volume(12, 12, 13), NormalizationMode.Spatial, ActivationType.ReLU),
-                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
+                gpu = new CuDnnBatchNormalizationLayer(cpu.InputInfo, NormalizationMode.Spatial, cpu.Weights, cpu.Biases, cpu.Iteration, cpu.Mu.AsSpan().Copy(), cpu.Sigma2.AsSpan().Copy(), cpu.ActivationType);
             TestBackward(cpu, gpu, 400);
         }
 

Unit/NeuralNetwork.NET.Unit/SerializationTest.cs

@@ -69,7 +69,8 @@ public void NetworkSerialization()
                 NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
                 NetworkLayers.Pooling(ActivationType.ReLU),
                 NetworkLayers.Convolutional((10, 10), 20, ActivationType.Identity),
-                NetworkLayers.Pooling(ActivationType.ReLU),
+                NetworkLayers.Pooling(ActivationType.Identity),
+                NetworkLayers.BatchNormalization(NormalizationMode.Spatial, ActivationType.ReLU),
                 NetworkLayers.FullyConnected(125, ActivationType.Tanh),
                 NetworkLayers.Softmax(133));
             using (MemoryStream stream = new MemoryStream())