Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 6579)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkClassification.cs	(revision 6580)
@@ -38,5 +38,5 @@
   /// Neural network classification data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Neural Network Classification", "Neural network classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
   [Creatable("Data Analysis")]
   [StorableClass]
@@ -156,14 +156,4 @@
         throw new NotSupportedException("Neural network classification does not support NaN or infinity values in the input dataset.");
 
-      alglib.multilayerperceptron multiLayerPerceptron = null;
-      int numberOfClasses = problemData.ClassValues.Count();
-      if (nLayers == 0) {
-        alglib.mlpcreatec0(allowedInputVariables.Count(), numberOfClasses, out multiLayerPerceptron);
-      } else if (nLayers == 1) {
-        alglib.mlpcreatec1(allowedInputVariables.Count(), nHiddenNodes1, numberOfClasses, out multiLayerPerceptron);
-      } else if (nLayers == 2) {
-        alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, numberOfClasses, out multiLayerPerceptron);
-      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
-      alglib.mlpreport rep;
       int nRows = inputMatrix.GetLength(0);
       int nFeatures = inputMatrix.GetLength(1) - 1;
@@ -178,4 +168,14 @@
         inputMatrix[row, nFeatures] = classIndizes[inputMatrix[row, nFeatures]];
       }
+
+      alglib.multilayerperceptron multiLayerPerceptron = null;
+      if (nLayers == 0) {
+        alglib.mlpcreatec0(allowedInputVariables.Count(), nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 1) {
+        alglib.mlpcreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, out multiLayerPerceptron);
+      } else if (nLayers == 2) {
+        alglib.mlpcreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, out multiLayerPerceptron);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
 
       int info;
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 6580)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassification.cs	(revision 6580)
@@ -0,0 +1,205 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble classification data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Classification", "Neural network ensemble classification data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleClassification : FixedDataAnalysisAlgorithm<IClassificationProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleClassificationModelResultName = "Neural network ensemble classification solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public ConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (ConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassification(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleClassification(NeuralNetworkEnsembleClassification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassification()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { new IntValue(0), new IntValue(1), new IntValue(2) });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      Problem = new ClassificationProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassification(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run() {
+      double rmsError, avgRelError, relClassError;
+      var solution = CreateNeuralNetworkEnsembleClassificationSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError, out relClassError);
+      Results.Add(new Result(NeuralNetworkEnsembleClassificationModelResultName, "The neural network ensemble classification solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble regression solution on the training set.", new PercentValue(avgRelError)));
+      Results.Add(new Result("Relative classification error", "The percentage of misclassified samples.", new PercentValue(relClassError)));
+    }
+
+    public static IClassificationSolution CreateNeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError, out double relClassError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndizes;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network ensemble classification does not support NaN or infinity values in the input dataset.");
+
+      int nRows = inputMatrix.GetLength(0);
+      int nFeatures = inputMatrix.GetLength(1) - 1;
+      double[] classValues = dataset.GetVariableValues(targetVariable).Distinct().OrderBy(x => x).ToArray();
+      int nClasses = classValues.Count();
+      // map original class values to values [0..nClasses-1]
+      Dictionary<double, double> classIndizes = new Dictionary<double, double>();
+      for (int i = 0; i < nClasses; i++) {
+        classIndizes[classValues[i]] = i;
+      }
+      for (int row = 0; row < nRows; row++) {
+        inputMatrix[row, nFeatures] = classIndizes[inputMatrix[row, nFeatures]];
+      }
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreatec0(allowedInputVariables.Count(), nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreatec1(allowedInputVariables.Count(), nHiddenNodes1, nClasses, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreatec2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, nClasses, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble regression solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+      relClassError = alglib.mlperelclserror(mlpEnsemble, inputMatrix, nRows);
+
+      return new NeuralNetworkEnsembleClassificationSolution(problemData, new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables, problemData.ClassValues.ToArray()));
+    }
+    #endregion
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 6580)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleClassificationSolution.cs	(revision 6580)
@@ -0,0 +1,57 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleClassificationSolution", "Represents a neural network ensemble solution for a classification problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleClassificationSolution : ClassificationSolution, INeuralNetworkEnsembleClassificationSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleClassificationSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleClassificationSolution(NeuralNetworkEnsembleClassificationSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleClassificationSolution(IClassificationProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleClassificationSolution(this, cloner);
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 6580)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleModel.cs	(revision 6580)
@@ -0,0 +1,324 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using SVM;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensembel model for regression and classification
+  /// </summary>
+  [StorableClass]
+  [Item("NeuralNetworkEnsembleModel", "Represents a neural network ensemble for regression and classification.")]
+  public sealed class NeuralNetworkEnsembleModel : NamedItem, INeuralNetworkEnsembleModel {
+
+    private alglib.mlpensemble mlpEnsemble;
+    public alglib.mlpensemble MultiLayerPerceptronEnsemble {
+      get { return mlpEnsemble; }
+      set {
+        if (value != mlpEnsemble) {
+          if (value == null) throw new ArgumentNullException();
+          mlpEnsemble = value;
+          OnChanged(EventArgs.Empty);
+        }
+      }
+    }
+
+    [Storable]
+    private string targetVariable;
+    [Storable]
+    private string[] allowedInputVariables;
+    [Storable]
+    private double[] classValues;
+    [StorableConstructor]
+    private NeuralNetworkEnsembleModel(bool deserializing)
+      : base(deserializing) {
+      if (deserializing)
+        mlpEnsemble = new alglib.mlpensemble();
+    }
+    private NeuralNetworkEnsembleModel(NeuralNetworkEnsembleModel original, Cloner cloner)
+      : base(original, cloner) {
+      mlpEnsemble = new alglib.mlpensemble();
+      mlpEnsemble.innerobj.columnmeans = (double[])original.mlpEnsemble.innerobj.columnmeans.Clone();
+      mlpEnsemble.innerobj.columnsigmas = (double[])original.mlpEnsemble.innerobj.columnsigmas.Clone();
+      mlpEnsemble.innerobj.dfdnet = (double[])original.mlpEnsemble.innerobj.dfdnet.Clone();
+      mlpEnsemble.innerobj.ensemblesize = original.mlpEnsemble.innerobj.ensemblesize;
+      mlpEnsemble.innerobj.issoftmax = original.mlpEnsemble.innerobj.issoftmax;
+      mlpEnsemble.innerobj.neurons = (double[])original.mlpEnsemble.innerobj.neurons.Clone();
+      mlpEnsemble.innerobj.nin = original.mlpEnsemble.innerobj.nin;
+      mlpEnsemble.innerobj.nout = original.mlpEnsemble.innerobj.nout;
+      mlpEnsemble.innerobj.postprocessing = original.mlpEnsemble.innerobj.postprocessing;
+      mlpEnsemble.innerobj.serializedlen = original.mlpEnsemble.innerobj.serializedlen;
+      mlpEnsemble.innerobj.serializedmlp = (double[])original.mlpEnsemble.innerobj.serializedmlp.Clone();
+      mlpEnsemble.innerobj.structinfo = (int[])original.mlpEnsemble.innerobj.structinfo.Clone();
+      mlpEnsemble.innerobj.tmpmeans = (double[])original.mlpEnsemble.innerobj.tmpmeans.Clone();
+      mlpEnsemble.innerobj.tmpsigmas = (double[])original.mlpEnsemble.innerobj.tmpsigmas.Clone();
+      mlpEnsemble.innerobj.tmpweights = (double[])original.mlpEnsemble.innerobj.tmpweights.Clone();
+      mlpEnsemble.innerobj.wcount = original.mlpEnsemble.innerobj.wcount;
+      mlpEnsemble.innerobj.weights = (double[])original.mlpEnsemble.innerobj.weights.Clone();
+      mlpEnsemble.innerobj.y = (double[])original.mlpEnsemble.innerobj.y.Clone();
+      targetVariable = original.targetVariable;
+      allowedInputVariables = (string[])original.allowedInputVariables.Clone();
+      if (original.classValues != null)
+        this.classValues = (double[])original.classValues.Clone();
+    }
+    public NeuralNetworkEnsembleModel(alglib.mlpensemble mlpEnsemble, string targetVariable, IEnumerable<string> allowedInputVariables, double[] classValues = null)
+      : base() {
+      this.name = ItemName;
+      this.description = ItemDescription;
+      this.mlpEnsemble = mlpEnsemble;
+      this.targetVariable = targetVariable;
+      this.allowedInputVariables = allowedInputVariables.ToArray();
+      if (classValues != null)
+        this.classValues = (double[])classValues.Clone();
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleModel(this, cloner);
+    }
+
+    public IEnumerable<double> GetEstimatedValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[1];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        yield return y[0];
+      }
+    }
+
+    public IEnumerable<double> GetEstimatedClassValues(Dataset dataset, IEnumerable<int> rows) {
+      double[,] inputData = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables, rows);
+
+      int n = inputData.GetLength(0);
+      int columns = inputData.GetLength(1);
+      double[] x = new double[columns];
+      double[] y = new double[classValues.Length];
+
+      for (int row = 0; row < n; row++) {
+        for (int column = 0; column < columns; column++) {
+          x[column] = inputData[row, column];
+        }
+        alglib.mlpeprocess(mlpEnsemble, x, ref y);
+        // find class for with the largest probability value
+        int maxProbClassIndex = 0;
+        double maxProb = y[0];
+        for (int i = 1; i < y.Length; i++) {
+          if (maxProb < y[i]) {
+            maxProb = y[i];
+            maxProbClassIndex = i;
+          }
+        }
+        yield return classValues[maxProbClassIndex];
+      }
+    }
+
+    #region events
+    public event EventHandler Changed;
+    private void OnChanged(EventArgs e) {
+      var handlers = Changed;
+      if (handlers != null)
+        handlers(this, e);
+    }
+    #endregion
+
+    #region persistence
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnMeans {
+      get {
+        return mlpEnsemble.innerobj.columnmeans;
+      }
+      set {
+        mlpEnsemble.innerobj.columnmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleColumnSigmas {
+      get {
+        return mlpEnsemble.innerobj.columnsigmas;
+      }
+      set {
+        mlpEnsemble.innerobj.columnsigmas = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleDfdnet {
+      get {
+        return mlpEnsemble.innerobj.dfdnet;
+      }
+      set {
+        mlpEnsemble.innerobj.dfdnet = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleSize {
+      get {
+        return mlpEnsemble.innerobj.ensemblesize;
+      }
+      set {
+        mlpEnsemble.innerobj.ensemblesize = value;
+      }
+    }
+    [Storable]
+    private bool MultiLayerPerceptronEnsembleIsSoftMax {
+      get {
+        return mlpEnsemble.innerobj.issoftmax;
+      }
+      set {
+        mlpEnsemble.innerobj.issoftmax = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleNeurons {
+      get {
+        return mlpEnsemble.innerobj.neurons;
+      }
+      set {
+        mlpEnsemble.innerobj.neurons = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleNin {
+      get {
+        return mlpEnsemble.innerobj.nin;
+      }
+      set {
+        mlpEnsemble.innerobj.nin = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleNout {
+      get {
+        return mlpEnsemble.innerobj.nout;
+      }
+      set {
+        mlpEnsemble.innerobj.nout = value;
+      }
+    }
+    [Storable]
+    private bool MultiLayerPerceptronEnsemblePostprocessing {
+      get {
+        return mlpEnsemble.innerobj.postprocessing;
+      }
+      set {
+        mlpEnsemble.innerobj.postprocessing = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleSerializedLen {
+      get {
+        return mlpEnsemble.innerobj.serializedlen;
+      }
+      set {
+        mlpEnsemble.innerobj.serializedlen = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleSerializedMlp {
+      get {
+        return mlpEnsemble.innerobj.serializedmlp;
+      }
+      set {
+        mlpEnsemble.innerobj.serializedmlp = value;
+      }
+    }
+    [Storable]
+    private int[] MultiLayerPerceptronStuctinfo {
+      get {
+        return mlpEnsemble.innerobj.structinfo;
+      }
+      set {
+        mlpEnsemble.innerobj.structinfo = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleTmpMeans {
+      get {
+        return mlpEnsemble.innerobj.tmpmeans;
+      }
+      set {
+        mlpEnsemble.innerobj.tmpmeans = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleTmpSigmas {
+      get {
+        return mlpEnsemble.innerobj.tmpsigmas;
+      }
+      set {
+        mlpEnsemble.innerobj.tmpsigmas = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronEnsembleTmpWeights {
+      get {
+        return mlpEnsemble.innerobj.tmpweights;
+      }
+      set {
+        mlpEnsemble.innerobj.tmpweights = value;
+      }
+    }
+    [Storable]
+    private int MultiLayerPerceptronEnsembleWCount {
+      get {
+        return mlpEnsemble.innerobj.wcount;
+      }
+      set {
+        mlpEnsemble.innerobj.wcount = value;
+      }
+    }
+
+    [Storable]
+    private double[] MultiLayerPerceptronWeights {
+      get {
+        return mlpEnsemble.innerobj.weights;
+      }
+      set {
+        mlpEnsemble.innerobj.weights = value;
+      }
+    }
+    [Storable]
+    private double[] MultiLayerPerceptronY {
+      get {
+        return mlpEnsemble.innerobj.y;
+      }
+      set {
+        mlpEnsemble.innerobj.y = value;
+      }
+    }
+    #endregion
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 6580)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegression.cs	(revision 6580)
@@ -0,0 +1,191 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Neural network ensemble regression data analysis algorithm.
+  /// </summary>
+  [Item("Neural Network Ensemble Regression", "Neural network ensemble regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/mlpensembles.php")]
+  [Creatable("Data Analysis")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
+    private const string EnsembleSizeParameterName = "EnsembleSize";
+    private const string DecayParameterName = "Decay";
+    private const string HiddenLayersParameterName = "HiddenLayers";
+    private const string NodesInFirstHiddenLayerParameterName = "NodesInFirstHiddenLayer";
+    private const string NodesInSecondHiddenLayerParameterName = "NodesInSecondHiddenLayer";
+    private const string RestartsParameterName = "Restarts";
+    private const string NeuralNetworkEnsembleRegressionModelResultName = "Neural network ensemble regression solution";
+
+    #region parameter properties
+    public IFixedValueParameter<IntValue> EnsembleSizeParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EnsembleSizeParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> DecayParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[DecayParameterName]; }
+    }
+    public ConstrainedValueParameter<IntValue> HiddenLayersParameter {
+      get { return (ConstrainedValueParameter<IntValue>)Parameters[HiddenLayersParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInFirstHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInFirstHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> NodesInSecondHiddenLayerParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[NodesInSecondHiddenLayerParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EnsembleSize {
+      get { return EnsembleSizeParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of models in the ensemble must be positive and at least one.", "EnsembleSize");
+        EnsembleSizeParameter.Value.Value = value;
+      }
+    }
+    public double Decay {
+      get { return DecayParameter.Value.Value; }
+      set {
+        if (value < 0.001 || value > 100) throw new ArgumentException("The decay parameter should be set to a value between 0.001 and 100.", "Decay");
+        DecayParameter.Value.Value = value;
+      }
+    }
+    public int HiddenLayers {
+      get { return HiddenLayersParameter.Value.Value; }
+      set {
+        if (value < 0 || value > 2) throw new ArgumentException("The number of hidden layers should be set to 0, 1, or 2.", "HiddenLayers");
+        HiddenLayersParameter.Value = (from v in HiddenLayersParameter.ValidValues
+                                       where v.Value == value
+                                       select v)
+                                      .Single();
+      }
+    }
+    public int NodesInFirstHiddenLayer {
+      get { return NodesInFirstHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first hidden layer must be at least one.", "NodesInFirstHiddenLayer");
+        NodesInFirstHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int NodesInSecondHiddenLayer {
+      get { return NodesInSecondHiddenLayerParameter.Value.Value; }
+      set {
+        if (value < 1) throw new ArgumentException("The number of nodes in the first second layer must be at least one.", "NodesInSecondHiddenLayer");
+        NodesInSecondHiddenLayerParameter.Value.Value = value;
+      }
+    }
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set {
+        if (value < 0) throw new ArgumentException("The number of restarts must be positive.", "Restarts");
+        RestartsParameter.Value.Value = value;
+      }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegression(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleRegression(NeuralNetworkEnsembleRegression original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegression()
+      : base() {
+      var validHiddenLayerValues = new ItemSet<IntValue>(new IntValue[] { new IntValue(0), new IntValue(1), new IntValue(2) });
+      var selectedHiddenLayerValue = (from v in validHiddenLayerValues
+                                      where v.Value == 1
+                                      select v)
+                                     .Single();
+      Parameters.Add(new FixedValueParameter<IntValue>(EnsembleSizeParameterName, "The number of simple neural network models in the ensemble. A good value is 10.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DecayParameterName, "The decay parameter for the training phase of the neural network. This parameter determines the strengh of regularization and should be set to a value between 0.001 (weak regularization) to 100 (very strong regularization). The correct value should be determined via cross-validation.", new DoubleValue(0.001)));
+      Parameters.Add(new ConstrainedValueParameter<IntValue>(HiddenLayersParameterName, "The number of hidden layers for the neural network (0, 1, or 2)", validHiddenLayerValues, selectedHiddenLayerValue));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInFirstHiddenLayerParameterName, "The number of nodes in the first hidden layer. The value should be rather large (30-100 nodes) in order to make the network highly flexible and run into the early stopping criterion). This value is not used if the number of hidden layers is zero.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(NodesInSecondHiddenLayerParameterName, "The number of nodes in the second hidden layer. This value is not used if the number of hidden layers is zero or one.", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of restarts for learning.", new IntValue(2)));
+
+      Problem = new RegressionProblem();
+    }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegression(this, cloner);
+    }
+
+    #region neural network ensemble
+    protected override void Run() {
+      double rmsError, avgRelError;
+      var solution = CreateNeuralNetworkEnsembleRegressionSolution(Problem.ProblemData, EnsembleSize, HiddenLayers, NodesInFirstHiddenLayer, NodesInSecondHiddenLayer, Decay, Restarts, out rmsError, out avgRelError);
+      Results.Add(new Result(NeuralNetworkEnsembleRegressionModelResultName, "The neural network ensemble regression solution.", solution));
+      Results.Add(new Result("Root mean square error", "The root of the mean of squared errors of the neural network ensemble regression solution on the training set.", new DoubleValue(rmsError)));
+      Results.Add(new Result("Average relative error", "The average of relative errors of the neural network ensemble regression solution on the training set.", new PercentValue(avgRelError)));
+    }
+
+    public static IRegressionSolution CreateNeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, int ensembleSize, int nLayers, int nHiddenNodes1, int nHiddenNodes2, double decay, int restarts,
+      out double rmsError, out double avgRelError) {
+      Dataset dataset = problemData.Dataset;
+      string targetVariable = problemData.TargetVariable;
+      IEnumerable<string> allowedInputVariables = problemData.AllowedInputVariables;
+      IEnumerable<int> rows = problemData.TrainingIndizes;
+      double[,] inputMatrix = AlglibUtil.PrepareInputMatrix(dataset, allowedInputVariables.Concat(new string[] { targetVariable }), rows);
+      if (inputMatrix.Cast<double>().Any(x => double.IsNaN(x) || double.IsInfinity(x)))
+        throw new NotSupportedException("Neural network ensemble regression does not support NaN or infinity values in the input dataset.");
+
+      alglib.mlpensemble mlpEnsemble = null;
+      if (nLayers == 0) {
+        alglib.mlpecreate0(allowedInputVariables.Count(), 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 1) {
+        alglib.mlpecreate1(allowedInputVariables.Count(), nHiddenNodes1, 1, ensembleSize, out mlpEnsemble);
+      } else if (nLayers == 2) {
+        alglib.mlpecreate2(allowedInputVariables.Count(), nHiddenNodes1, nHiddenNodes2, 1, ensembleSize, out mlpEnsemble);
+      } else throw new ArgumentException("Number of layers must be zero, one, or two.", "nLayers");
+      alglib.mlpreport rep;
+      int nRows = inputMatrix.GetLength(0);
+
+      int info;
+      alglib.mlpetraines(mlpEnsemble, inputMatrix, nRows, decay, restarts, out info, out rep);
+      if (info != 6) throw new ArgumentException("Error in calculation of neural network ensemble regression solution");
+
+      rmsError = alglib.mlpermserror(mlpEnsemble, inputMatrix, nRows);
+      avgRelError = alglib.mlpeavgrelerror(mlpEnsemble, inputMatrix, nRows);
+
+      return new NeuralNetworkEnsembleRegressionSolution(problemData, new NeuralNetworkEnsembleModel(mlpEnsemble, targetVariable, allowedInputVariables));
+    }
+    #endregion
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 6580)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkEnsembleRegressionSolution.cs	(revision 6580)
@@ -0,0 +1,57 @@
+#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+  /// <summary>
+  /// Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.
+  /// </summary>
+  [Item("NeuralNetworkEnsembleRegressionSolution", "Represents a neural network ensemble solution for a regression problem which can be visualized in the GUI.")]
+  [StorableClass]
+  public sealed class NeuralNetworkEnsembleRegressionSolution : RegressionSolution, INeuralNetworkEnsembleRegressionSolution {
+
+    public new INeuralNetworkEnsembleModel Model {
+      get { return (INeuralNetworkEnsembleModel)base.Model; }
+      set { base.Model = value; }
+    }
+
+    [StorableConstructor]
+    private NeuralNetworkEnsembleRegressionSolution(bool deserializing) : base(deserializing) { }
+    private NeuralNetworkEnsembleRegressionSolution(NeuralNetworkEnsembleRegressionSolution original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public NeuralNetworkEnsembleRegressionSolution(IRegressionProblemData problemData, INeuralNetworkEnsembleModel nnModel)
+      : base(nnModel, problemData) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NeuralNetworkEnsembleRegressionSolution(this, cloner);
+    }
+  }
+}
Index: trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs
===================================================================
--- trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 6579)
+++ trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/NeuralNetwork/NeuralNetworkRegression.cs	(revision 6580)
@@ -38,5 +38,5 @@
   /// Neural network regression data analysis algorithm.
   /// </summary>
-  [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB).")]
+  [Item("Neural Network Regression", "Neural network regression data analysis algorithm (wrapper for ALGLIB). Further documentation: http://www.alglib.net/dataanalysis/neuralnetworks.php")]
   [Creatable("Data Analysis")]
   [StorableClass]
