Changeset 6579

Timestamp:

07/21/11 11:45:19 (13 years ago)

Author:

gkronber

Message:

#1474: implemented neural networks for classification.

Location:

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (2 diffs)
• Interfaces/INeuralNetworkClassificationSolution.cs (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkRegressionSolution.cs) (1 diff)
• NeuralNetwork/NeuralNetworkClassification.cs (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs) (6 diffs)
• NeuralNetwork/NeuralNetworkClassificationSolution.cs (copied) (copied from trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegressionSolution.cs) (2 diffs)
• NeuralNetwork/NeuralNetworkRegression.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -112 +112 @@
     <Compile Include="HeuristicLabAlgorithmsDataAnalysisPlugin.cs" />
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
+    <Compile Include="Interfaces\INeuralNetworkClassificationSolution.cs" />
     <Compile Include="Interfaces\INeuralNetworkRegressionSolution.cs" />
     <Compile Include="Interfaces\INeuralNetworkModel.cs" />
@@ -134 +135 @@
     <Compile Include="Linear\MultinomialLogitClassificationSolution.cs" />
     <Compile Include="Linear\MultinomialLogitModel.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassification.cs" />
+    <Compile Include="NeuralNetwork\NeuralNetworkClassificationSolution.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkModel.cs" />
     <Compile Include="NeuralNetwork\NeuralNetworkRegression.cs" />

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/Interfaces/INeuralNetworkClassificationSolution.cs

@@ -26 +26 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   /// <summary>
-  /// Interface to represent a neural network regression solution
+  /// Interface to represent a neural network classification solution
   /// </summary>
-  public interface INeuralNetworkRegressionSolution : IRegressionSolution {
+  public interface INeuralNetworkClassificationSolution : IClassificationSolution {
     new INeuralNetworkModel Model { get; }
   }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs

@@ -36 +36 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   /// <summary>
-  /// Neural network regression data analysis algorithm.
+  /// Neural network classification data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB).")]
   [Creatable("Data Analysis")]
   [StorableClass]
-  public sealed class NeuralNetworkRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+  public sealed class NeuralNetworkClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
     private const string DecayParameterName = "Decay";
     private const string HiddenLayersParameterName = "HiddenLayers";
@@ -47 +47 @@
     private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
     private const string RestartsParameterName = "Restarts";
-    private const string NeuralNetworkRegressionModelResultName = "Neural network regression solution";
+    private const string NeuralNetworkRegressionModelResultName = "Neural network classification solution";
 
     #region parameter properties
@@ -110 +110 @@
 
     [StorableConstructor]
-    private NeuralNetworkRegression(bool deserializing) : base(deserializing) { }
-    private NeuralNetworkRegression(NeuralNetworkRegression original, Cloner cloner)
+    private NeuralNetworkClassification(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkClassification(NeuralNetworkClassification original, Cloner cloner)
       : base(original, cloner) {
     }
-    public NeuralNetworkRegression()
+    public NeuralNetworkClassification()
       : base() {
       var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { new IntValue(0), new IntValue(1), new IntValue(2) });
@@ -127 +127 @@
       Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
 
-      Problem = new RegressionProblem();
+      Problem = new ClassificationProblem();
     }
     [StorableHook(HookType.AfterDeserialization)]
@@ -133 +133 @@
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new NeuralNetworkRegression(this, cloner);
+      return new NeuralNetworkClassification(this, cloner);
     }
 
     #region neural network
     protected override void Run() {
-      double rmsError, avgRelError;
-      var solution = CreateNeuralNetworkRegressionSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkClassificationSolution(Problem.ProblemData, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
       Results.Add(new Result(NeuralNetworkRegressionModelResultName, "The neural network regression solution.", solution));
       Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network regression solution on the training set.", new DoubleValue(rmsError)));
       Results.Add(new Result("Average relative error", "The average of relative errors of the neural network regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
     }
 
-    public static IRegressionSolution CreateNeuralNetworkRegressionSolution(IRegressionProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
-      out double rmsError, out double avgRelError) {
+    public static IClassificationSolution CreateNeuralNetworkClassificationSolution(IClassificationProblemData problemData, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
       Dataset dataset = problemData.Dataset;
       string targetVariable = problemData.TargetVariable;
@@ -153 +154 @@
       double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
       if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
-        throw new NotSupportedException("Neural network regression does not support NaN or infinity values in the input dataset.");
-
-      double targetMin = problemData.Dataset.GetEnumeratedVariableValues(targetVariable).Min();
-      targetMin = targetMin - targetMin * 0.1; // -10%
-      double targetMax = problemData.Dataset.GetEnumeratedVariableValues(targetVariable).Max();
-      targetMax = targetMax + targetMax * 0.1; // + 10%
+        throw new NotSupportedException("Neural network classification does not support NaN or infinity values in the input dataset.");
 
       alglib.multilayerperceptron multiLayerPerceptron = null;
+      int numberOfClasses = problemData.ClassValues.Count();
       if (nLayers == 0) {
-        alglib.mlpcreater0(allowedInputVariables.Count(), 1, targetMin, targetMax, out multiLayerPerceptron);
+        alglib.mlpcreatec0(allowedInputVariables.Count(), numberOfClasses, out multiLayerPerceptron);
       } else if (nLayers == 1) {
-        alglib.mlpcreater1(allowedInputVariables.Count(), nHiddenNodes1, 1, targetMin, targetMax, out multiLayerPerceptron);
+        alglib.mlpcreatec1(allowedInputVariables.Count(), nHiddenNodes1, numberOfClasses, out multiLayerPerceptron);
       } else if (nLayers == 2) {
-        alglib.mlpcreater2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, targetMin, targetMax, out multiLayerPerceptron);
+        alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, numberOfClasses, out multiLayerPerceptron);
       } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
       alglib.mlpreport rep;
       int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetVariableValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndizes = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndizes[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndizes[inputMatrix[row, nFeatures]];
+      }
 
       int info;
       // using mlptrainlm instead of mlptraines or mlptrainbfgs because only one parameter is necessary
       alglib.mlptrainlm(multiLayerPerceptron, inputMatrix, nRows, decay, restarts, out info, out rep);
-      if (info != 2) throw new ArgumentException("Error in calculation of neural network regression solution");
+      if (info != 2) throw new ArgumentException("Error in calculation of neural network classification solution");
 
       rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
       avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+      relClassError = alglib.mlpclserror(multiLayerPerceptron, inputMatrix, nRows) / (double)nRows;
 
-      return new NeuralNetworkRegressionSolution(problemData, new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables));
+      return new NeuralNetworkClassificationSolution(problemData, new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables, problemData.ClassValues.ToArray()));
     }
     #endregion

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassificationSolution.cs

@@ -31 +31 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   /// <summary>
-  /// Represents a neural network solution for a regression problem which can be visualized in the GUI.
+  /// Represents a neural network solution for a classification problem which can be visualized in the GUI.
   /// </summary>
-  [Item("NeuralNetworkRegressionSolution", "Represents a neural network solution for a regression problem which can be visualized in the GUI.")]
+  [Item("NeuralNetworkClassificationSolution", "Represents a neural network solution for a classification problem which can be visualized in the GUI.")]
   [StorableClass]
-  public sealed class NeuralNetworkRegressionSolution : RegressionSolution, INeuralNetworkRegressionSolution {
+  public sealed class NeuralNetworkClassificationSolution : ClassificationSolution, INeuralNetworkClassificationSolution {
 
     public new INeuralNetworkModel Model {
@@ -43 +43 @@
 
     [StorableConstructor]
-    private NeuralNetworkRegressionSolution(bool deserializing) : base(deserializing) { }
-    private NeuralNetworkRegressionSolution(NeuralNetworkRegressionSolution original, Cloner cloner)
+    private NeuralNetworkClassificationSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkClassificationSolution(NeuralNetworkClassificationSolution original, Cloner cloner)
       : base(original, cloner) {
     }
-    public NeuralNetworkRegressionSolution(IRegressionProblemData problemData, INeuralNetworkModel nnModel)
+    public NeuralNetworkClassificationSolution(IClassificationProblemData problemData, INeuralNetworkModel nnModel)
       : base(nnModel, problemData) {
     }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new NeuralNetworkRegressionSolution(this, cloner);
+      return new NeuralNetworkClassificationSolution(this, cloner);
     }
   }

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs

@@ -177 +177 @@
 
       rmsError = alglib.mlprmserror(multiLayerPerceptron, inputMatrix, nRows);
-      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);
+      avgRelError = alglib.mlpavgrelerror(multiLayerPerceptron, inputMatrix, nRows);      
 
       return new NeuralNetworkRegressionSolution(problemData, new NeuralNetworkModel(multiLayerPerceptron, targetVariable, allowedInputVariables));